ton/SQLLLM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat lamaiin
2024-10-16 13:10:31 +07:00
parent a711c0c111
commit 159d1717a4
2 changed files with 181 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

377
src/interface.jl
View File

@@ -158,6 +158,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
- If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
- If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query". - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
- Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
- Do not use backticks (`). Use double quotes instead.
You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action: You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action:
1) Understanding: 1) Understanding:
@@ -183,7 +184,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
Let's begin! Let's begin!
""" """
# [WORKING] add no backtick
workprogress = "" workprogress = ""
for (k, v) in state[:thoughtHistory] for (k, v) in state[:thoughtHistory]
if k ∉ [:query] if k ∉ [:query]
@@ -191,31 +192,37 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
end end
end end
usermsg =
"""
$(context[:tablelist])
User query: $(state[:thoughtHistory][:question])
Hints: $similarSQL
Your Q&A: $QandA
Your work progress: $workprogress
Evaluation: $(state[:evaluation])
Suggestion: $(state[:suggestion])
"""
_prompt =
[
Dict(:name=> "system", :text=> systemmsg),
Dict(:name=> "user", :text=> usermsg)
]
# put in model format
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
prompt *=
"""
<|start_header_id|>assistant<|end_header_id|>
"""
response = nothing # store for show when error msg show up response = nothing # store for show when error msg show up
errornote = ""
noise = ""
for attempt in 1:10 for attempt in 1:10
usermsg =
"""
$(context[:tablelist])
User query: $(state[:thoughtHistory][:question])
Hints: $similarSQL
Your Q&A: $QandA
Your work progress: $workprogress
Evaluation: $(state[:evaluation])
Suggestion: $(state[:suggestion])
$errornote
$noise
"""
_prompt =
[
Dict(:name=> "system", :text=> systemmsg),
Dict(:name=> "user", :text=> usermsg)
]
# put in model format
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
prompt *=
"""
<|start_header_id|>assistant<|end_header_id|>
"""
try try
response = text2textInstructLLM(prompt) response = text2textInstructLLM(prompt)
@@ -226,6 +233,10 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
delete!(responsedict, :observation) delete!(responsedict, :observation)
if occursin('`', response)
response = replace(response, '`'=>"")
end
toollist = ["TABLEINFO", "GETDATA"] toollist = ["TABLEINFO", "GETDATA"]
if responsedict[:action_name] ∉ toollist if responsedict[:action_name] ∉ toollist
error("SQL decisionMaker() didn't use the given functions ", @__FILE__, " ", @__LINE__) error("SQL decisionMaker() didn't use the given functions ", @__FILE__, " ", @__LINE__)
@@ -258,220 +269,175 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
errorMsg = String(take!(io)) errorMsg = String(take!(io))
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
println("") println("")
println("Attempt $attempt. Error occurred: $errorMsg\n$st") println("\n~~~ SQLLLM decisionMaker() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, " ", @__LINE__)
println("") println("")
noise = GeneralUtils.randstrings(3, 5)
end end
end end
error("DecisionMaker failed to generate a thought ", response) error("DecisionMaker failed to generate a thought ", response)
end end
# function decisionMaker(state::T1, context, text2textInstructLLM::Function,
# QandA::T2; similarSQL::Union{T3, Nothing}=nothing
# )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
# function decisionMaker(state::T2, config::T1 # similarSQL =
# )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:AbstractDict} # if similarSQL === nothing
# "None"
# else
# "This is the closest matching SQL statement for a similar query: $similarSQL"
# end
# if isfile("lesson.json") # # lessonDict =
# lessonDict = copy(JSON3.read("lesson.json")) # # if isfile("lesson.json")
# else # # lessonDict = copy(JSON3.read("lesson.json"))
# lessonDict = nothing # # else
# end # # lessonDict = nothing
# # end
# lesson = # # lessonDict = nothing
# if lessonDict === nothing
# ""
# else
# """
# You have attempted to help the user before and failed, either because your reasoning for the
# recommendation was incorrect or your response did not exactly match the user expectation.
# The following lesson(s) give a plan to avoid failing to help the user in the same way you
# did previously. Use them to improve your strategy to help the user.
# Here are some lessons in JSON format: # # lesson =
# $(JSON3.write(lessonDict)) # # if lessonDict === nothing
# # ""
# # else
# # """
# # You have attempted to help the user before and failed, either because your reasoning for the
# # recommendation was incorrect or your response did not exactly match the user expectation.
# # The following lesson(s) give a plan to avoid failing to help the user in the same way you
# # did previously. Use them to improve your strategy to help the user.
# When providing the thought and action for the current trial, that into account these failed # # Here are some lessons in JSON format:
# trajectories and make sure not to repeat the same mistakes and incorrect answers. # # $(JSON3.write(lessonDict))
# """
# end
# _prompt = # # When providing the thought and action for the current trial, that into account these failed
# # trajectories and make sure not to repeat the same mistakes and incorrect answers.
# # """
# # end
# systemmsg =
# """ # """
# You are a helpful data engineer. # You are a helpful assistant that get the data from a database to satisfy the user's query.
# Your goal is to help the user to get what the user wants. # You are also eager to improve your helpfulness.
# You are also keen to improve your helpfulness with lesson(s).
# You must follow the following criteria: # At each round of conversation, the user will give you the current situation:
# 1) Get to know what table are available in the database. # User Query: ...
# 2) Get to know what the data in the table looks like. # Hints: ...
# 3) If you can't find a single table that can be used to answer the user's question, try joining multiple tables to see if you can obtain the answer. # Your Q&A: ...
# 4) If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your question". # Your work progress: ...
# Evaluation: Evaluation of the latest action and observation
# Suggestion: ...
# You should only respond with interleaving Thought, Action, Observation steps. # You should consider the following guidelines:
# Thought can reason about the current situation, and Action can be one of the following functions: # - Do not create any table in the database
# 1) listalltables[NA], which you can use to list all tables in the database and see their descriptions. "NA" word is the function input. # - Column name can be the same in different tables. Refer to column comments to get more details by using TABLEINFO function
# 2) tableinfo[table_name], which you can use to see the table and its column description. "table_name" is name of the table you want to get info. # - A junction table can be used to link tables together. Another use case is for filtering data.
# 3) getdata[instructions], which you can use to ask other people to get the data from tables for you. "instructions" should clearly describe how you want others to extract the data. # - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
# For example, # - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
# a. Query the "Engine" table to identify the engine types that have 3 cylinders. This can be done using a SELECT statement in SQL, filtering the results where the number of cylinders equals 3. # - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
# b. Once you have identified the engine types with 3 cylinders, use this information to query the "Car" table. You're looking for car models that are associated with these engine types. This can be achieved by performing a JOIN operation between the "Car" and "Engine" tables based on the engine type.
# 4) finalanswerbox[answer], which returns your answer to the user. "answer" is your answer to the user question.
# After each observation, provide the next Thought and next Action.
# You should only respond in JSON format as describe below: # You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action:
# { # 1) Understanding:
# "thought": "your reasoning", # - State your understanding about the current situation.
# "action": {"name": "action to take", "input": "Action input"}, # 2) Reasoning:
# "observation": "result of the action" # - State your step by step reasoning about the current situation.
# } # 3) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
# 4) Action_name (Must be aligned with your plan): Can be one of the following functions:
# - TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. "list_of_table_name" is a list of table name you want to get info. e.g. TABLEINFO["table name 1", "table name 2"]
# - GETDATA[SQL], which you can use to get the data from the database. "SQL" is the single SQL command to be executed against the database.
# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
# 5) Action_input: Input to the action
# 6) Observation: Result of the immediately preceding action
# Here are some examples: # You should only respond in format as described below:
# { # Understanding: ...
# "question": "I would like to buy a sedan with 8 seats.", # Reasoning: ...
# "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.", # Plan: ...
# "action_1": {"name": "inventory", "input": "sedan with 8 seats."}, # Action_name: ...
# "observation_1": "Several model has 8 seats. Available color are black, red green" # Action_input: ...
# } # Observation: ...
# {
# "thought": "I have a few color for the user to choose from. I will ask him what color he likes.",
# "action": {"name": "chatbox", "input": "Which color do you like?"}
# "observation": "I'll take black."
# }
# $lesson
# Let's begin! # Let's begin!
# $(JSON3.write(state[:thoughtHistory]))
# {"thought"
# """ # """
# workprogress = ""
# # _prompt = # for (k, v) in state[:thoughtHistory]
# # """ # if k ∉ [:query]
# # You are a helpful data engineer. # workprogress *= "$k: $v\n"
# # Your goal is to help the user to get what the user wants. # end
# # You are also keen to improve your helpfulness with lesson(s).
# # You must follow the following criteria:
# # 1) Get to know what table are available in the database.
# # 2) Get to know what the data in the table looks like.
# # 3) If you can't find a single table that can be used to answer the user's question, try joining multiple tables to see if you can obtain the answer.
# # 4) Keep trying even if you get SQL execution error.
# # You should only respond with interleaving Thought, Action, Observation steps.
# # Thought can reason about the current situation, and Action can be one of the following functions:
# # 1) listalltables[NA], which you can use to list all tables in the database and see their descriptions. "NA" word is the function input.
# # 2) tableinfo[table_name], which you can use to see the table and its column description. "table_name" is name of the table you want to get info.
# # 3) getdata[SQL], which you can use to ask other people to get the data from tables for you. "SQL" is the command you will use to extract the data.
# # 4) finalanswerbox[answer], which returns your answer to the user. "answer" is your answer to the user question.
# # After each observation, provide the next Thought and next Action.
# # You should only respond in JSON format as describe below:
# # {
# # "thought": "your reasoning",
# # "action": {"name": "action to take", "input": "Action input"},
# # "observation": "result of the action"
# # }
# # Here are some examples:
# # {
# # "question": "I would like to buy a sedan with 8 seats.",
# # "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.",
# # "action_1": {"name": "inventory", "input": "sedan with 8 seats."},
# # "observation_1": "Several model has 8 seats. Available color are black, red green"
# # }
# # {
# # "thought": "I have a few color for the user to choose from. I will ask him what color he likes.",
# # "action": {"name": "chatbox", "input": "Which color do you like?"}
# # "observation": "I'll take black."
# # }
# # $lesson
# # Let's begin!
# # $(JSON3.write(state[:thoughtHistory]))
# # {"thought"
# # """
# # apply LLM specific instruct format
# externalService = config[:externalservice][:text2textinstruct]
# llminfo = externalService[:llminfo]
# prompt =
# if llminfo[:name] == "llama3instruct"
# GeneralUtils.formatLLMtext_llama3instruct("system", _prompt)
# else
# error("llm model name is not defied yet $(@__LINE__)")
# end # end
# msgMeta = GeneralUtils.generate_msgMeta( # usermsg =
# externalService[:mqtttopic], # """
# senderName= "decisionMaker", # $(context[:tablelist])
# senderId= string(uuid4()), # User query: $(state[:thoughtHistory][:question])
# receiverName= "text2textinstruct", # Hints: $similarSQL
# mqttBroker= config[:mqttServerInfo][:broker], # Your Q&A: $QandA
# mqttBrokerPort= config[:mqttServerInfo][:port], # Your work progress: $workprogress
# ) # Evaluation: $(state[:evaluation])
# Suggestion: $(state[:suggestion])
# """
# outgoingMsg = Dict( # _prompt =
# :msgMeta=> msgMeta, # [
# :payload=> Dict( # Dict(:name=> "system", :text=> systemmsg),
# :text=> prompt, # Dict(:name=> "user", :text=> usermsg)
# :kwargs=> Dict( # ]
# :max_tokens=> 512,
# :stop=> ["<|eot_id|>"],
# )
# )
# )
# @show outgoingMsg
# for attempt in 1:5 # # put in model format
# prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
# prompt *=
# """
# <|start_header_id|>assistant<|end_header_id|>
# """
# response = nothing # store for show when error msg show up
# for attempt in 1:10
# try # try
# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) # response = text2textInstructLLM(prompt)
# _responseJsonStr = response[:response][:text]
# expectedJsonExample =
# """
# Here is an expected JSON format:
# {
# "thought": "...",
# "action": {"name": "...", "input": "..."},
# "observation": "..."
# }
# """
# responseJsonStr = FormatCorrector.jsoncorrection(config, _responseJsonStr, expectedJsonExample)
# thoughtDict = copy(JSON3.read(responseJsonStr))
# # check if dict has all required value # # textToDict() search for action_input
# thought::AbstractString = thoughtDict[:thought] # responsedict = GeneralUtils.textToDict(response,
# actionname::AbstractString = thoughtDict[:action][:name] # ["Understanding", "Reasoning", "Plan", "Action_name", "Action_input", "Observation"],
# actioninput::AbstractString = thoughtDict[:action][:input] # rightmarker=":", symbolkey=true, lowercasekey=true)
# if actionname ∈ ["listalltables", "tableinfo", "getdata", "finalanswerbox"]
# # LLM use available function # delete!(responsedict, :observation)
# elseif thought == ""
# error("DecisionMaker has no thought") # toollist = ["TABLEINFO", "GETDATA"]
# elseif length(actioninput) == 0 # if responsedict[:action_name] ∉ toollist
# error("DecisionMaker has no actioninput") # error("SQL decisionMaker() didn't use the given functions ", @__FILE__, " ", @__LINE__)
# else # end
# error("DecisionMaker use wrong function")
# for i in toollist
# if occursin(i, responsedict[:action_input])
# error("Action_name is in action_input which is not allowed.")
# end
# end
# for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
# if length(JSON3.write(responsedict[i])) == 0
# error("$i is empty ", @__FILE__, " ", @__LINE__)
# end
# end # end
# # check if there are more than 1 key per categories # # check if there are more than 1 key per categories
# for i ∈ ["thought", "action", "observation"] # for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
# matchkeys = GeneralUtils.findMatchingDictKey(thoughtDict, i) # matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
# if length(matchkeys) > 1 # if length(matchkeys) > 1
# error("DecisionMaker has more than one key per categories") # error("DecisionMaker has more than one key per categories")
# end # end
# end # end
# return thoughtDict # return responsedict
# catch e # catch e
# io = IOBuffer() # io = IOBuffer()
# showerror(io, e) # showerror(io, e)
# errorMsg = String(take!(io)) # errorMsg = String(take!(io))
# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) # st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
# println("") # println("")
# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st" # println("Attempt $attempt. Error occurred: $errorMsg\n$st")
# println("") # println("")
# end # end
# end # end
# error("DecisionMaker failed to generate a thought") # error("DecisionMaker failed to generate a thought ", response)
# end # end
@@ -1221,6 +1187,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
- State your understanding about the current situation. - State your understanding about the current situation.
2) Q: Given the situation, "ask yourself" about the situation at least five, but no more than ten, questions. 2) Q: Given the situation, "ask yourself" about the situation at least five, but no more than ten, questions.
3) A: Given the situation, "answer to yourself" the best you can. 3) A: Given the situation, "answer to yourself" the best you can.
- Do not generate any text after the last answer.
You must only respond in format as described below: You must only respond in format as described below:
Understanding: ... Understanding: ...
@@ -1244,9 +1211,9 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
response = nothing # store for show when error msg show up response = nothing # store for show when error msg show up
errornote = "" errornote = ""
noise = ""
for attempt in 1:10 for attempt in 1:10
usermsg = usermsg =
""" """
$(context[:tablelist]) $(context[:tablelist])
@@ -1254,6 +1221,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
Hints: $similarSQL Hints: $similarSQL
Your work progress: $workprogress Your work progress: $workprogress
$errornote $errornote
$noise
""" """
_prompt = _prompt =
@@ -1271,10 +1239,18 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
try try
response = text2textInstructLLM(prompt) response = text2textInstructLLM(prompt)
# check if response is valid
q_number = count("Q", response) q_number = count("Q", response)
if q_number < 1 if q_number < 1
error("too few questions only $q_number questions are generated ", @__FILE__, " ", @__LINE__) errornote = "too few question"
error("too few questions only $q_number questions are generated")
end end
if occursin('`', response)
response = replace(response, '`'=>"")
end
# response = string(split(response, "Please")[1]) # LLM usually add comments which is no need. # response = string(split(response, "Please")[1]) # LLM usually add comments which is no need.
responsedict = GeneralUtils.textToDict(response, responsedict = GeneralUtils.textToDict(response,
["Understanding", "Q1"], ["Understanding", "Q1"],
@@ -1288,9 +1264,8 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
showerror(io, e) showerror(io, e)
errorMsg = String(take!(io)) errorMsg = String(take!(io))
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
println("") println("\n~~~ SQLLLM generatequestion() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, " ", @__LINE__)
println("Attempt $attempt. Error occurred: $errorMsg\n$st") noise = GeneralUtils.randstrings(3, 5)
println("")
end end
end end
error("generatequestion failed to generate a thought ", response) error("generatequestion failed to generate a thought ", response)

2
src/llmfunction.jl
View File

@@ -653,7 +653,7 @@ function SQLexecution(executeSQL::Function, sql::T
tablesize = size(df) tablesize = size(df)
row, column = tablesize row, column = tablesize
if row == 0 # if 0 row if row == 0 # if 0 row
error("The resulting table has 0 row. Possible causes: 1) You might be searching in the wrong place 2) There could be a typo in your search query 3) No data matches your search criteria.") error("The resulting table has 0 row. Possible causes: 1) You might be searching in the wrong place 2) There could be a typo in your search query.")
elseif column > 30 elseif column > 30
error("SQL execution failed. An unexpected error occurred. Please try again.") error("SQL execution failed. An unexpected error occurred. Please try again.")
end end