ton/SQLLLM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat lamaiin
2025-05-14 21:21:30 +07:00
parent aeda7e0baf
commit 8c5b1b6938
3 changed files with 83 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
src/interface.jl
View File

@@ -310,14 +310,14 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
toollist = ["TABLEINFO", "RUNSQL"] toollist = ["TABLEINFO", "RUNSQL"]
if responsedict[:action_name] ∉ toollist if responsedict[:action_name] ∉ toollist
errornote = "Your previous attempt has action_name that is not in the tool list" errornote = "Your previous attempt has action_name that is not in the tool list"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_name]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue continue
end end
for i in toollist for i in toollist
if occursin(i, responsedict[:action_input]) if occursin(i, responsedict[:action_input])
errornote = "Your previous attempt has action_name in action_input which is not allowed" errornote = "Your previous attempt has action_name in action_input which is not allowed"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_input]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue continue
end end
end end
@@ -330,39 +330,38 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
end end
end end
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict))
# store for later training # store for later training
responsedict[:thoughthistory] = state[:thoughtHistory] responsedict[:thoughthistory] = state[:thoughtHistory]
responsedict[:system] = systemmsg responsedict[:system] = systemmsg
responsedict[:prompt] = prompt responsedict[:prompt] = prompt
responsedict[:context] = context responsedict[:context] = context
responsedict[:think] = think responsedict[:think] = think
state[:decisionMaker] = responsedict
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # # read sessionId
pprintln(Dict(responsedict)) # sessionid = JSON3.read("/appfolder/app/sessionid.json")
# read sessionId # # save to filename ./log/decisionlog.txt
sessionid = JSON3.read("/appfolder/app/sessionid.json") # println("saving SQLLLM decisionMaker() to disk")
# filename = "agent_decision_log_$(sessionid[:id]).json"
# save to filename ./log/decisionlog.txt # filepath = "/appfolder/app/log/$filename"
println("saving SQLLLM decisionMaker() to disk") # # check whether there is a file path exists before writing to it
filename = "agent_decision_log_$(sessionid[:id]).json" # if !isfile(filepath)
filepath = "/appfolder/app/log/$filename" # decisionlist = [responsedict]
# check whether there is a file path exists before writing to it # println("Creating file $filepath")
if !isfile(filepath) # open(filepath, "w") do io
decisionlist = [responsedict] # JSON3.pretty(io, decisionlist)
println("Creating file $filepath") # end
open(filepath, "a") do io # else
JSON3.pretty(io, decisionlist) # # read the file and append new data
end # decisionlist = copy(JSON3.read(filepath))
else # push!(decisionlist, responsedict)
# read the file and append new data # println("Appending new data to file $filepath")
decisionlist = copy(JSON3.read(filepath)) # open(filepath, "w") do io
push!(decisionlist, responsedict) # JSON3.pretty(io, decisionlist)
println("Appending new data to file $filepath") # end
open(filepath, "w") do io # end
JSON3.pretty(io, decisionlist)
end
end
return responsedict return responsedict
end end
@@ -631,8 +630,6 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
Definitions: Definitions:
"question" is the user's question "question" is the user's question
"understanding" is agent's understanding about the current situation
"reasoning" is agent's step-by-step reasoning about the current situation
"plan" is agent's plan to complete the task from the current situation "plan" is agent's plan to complete the task from the current situation
"action_name" is the name of the action taken, which can be one of the following functions: "action_name" is the name of the action taken, which can be one of the following functions:
- RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
@@ -642,7 +639,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
"observation" is result of the preceding immediate action "observation" is result of the preceding immediate action
At each round of conversation, you will be given the following information: At each round of conversation, you will be given the following information:
Trajectory: ... Trajectory: A history of how you worked on the question chronologically.
You must follow the following guidelines: You must follow the following guidelines:
- When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
@@ -655,7 +652,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
- Do not generate additional thoughts or actions. - Do not generate additional thoughts or actions.
2) Answer_evaluation: 2) Answer_evaluation:
- Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No" 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no"
Bad example (The observation didn't answers the question): Bad example (The observation didn't answers the question):
question: Find cars with 4 wheels. question: Find cars with 4 wheels.
observation: There are an apple in the table. observation: There are an apple in the table.
@@ -670,7 +667,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
- 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question
- 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
- 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
5) Suggestion: what are the possible reason of this outcome, what can you learn from it and what suggestion can made? 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
You should only respond in format as described below: You should only respond in format as described below:
Trajectory_evaluation: ... Trajectory_evaluation: ...
@@ -748,9 +745,9 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
accepted_as_answer::AbstractString = responsedict[:accepted_as_answer] accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
if accepted_as_answer ∉ ["Yes", "No"] if accepted_as_answer ∉ ["yes", "no"]
errornote = "Your previous attempt's accepted_as_answer has wrong format" errornote = "Your previous attempt's accepted_as_answer has wrong format"
println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:accepted_as_answer]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue continue
end end
@@ -761,7 +758,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
state[:suggestion] = responsedict[:suggestion] state[:suggestion] = responsedict[:suggestion]
# mark as terminal state when the answer is achieved # mark as terminal state when the answer is achieved
if accepted_as_answer == "Yes" if accepted_as_answer ∈ ["Yes", "yes"]
# mark the state as terminal state because the evaluation say so. # mark the state as terminal state because the evaluation say so.
state[:isterminal] = true state[:isterminal] = true
@@ -770,10 +767,40 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
state[:reward] = responsedict[:score] state[:reward] = responsedict[:score]
end end
responsedict[:think] = think println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
println("\nEvaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict)) pprintln(Dict(responsedict))
# # store for later training
# responsedict[:thoughthistory] = state[:thoughtHistory]
# responsedict[:system] = systemmsg
# responsedict[:usermsg] = usermsg
# responsedict[:prompt] = prompt
# responsedict[:context] = context
# responsedict[:think] = think
# # read sessionId
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
# # save to filename ./log/decisionlog.txt
# println("saving SQLLLM evaluator() to disk")
# filename = "agent_evaluator_log_$(sessionid[:id]).json"
# filepath = "/appfolder/app/log/$filename"
# # check whether there is a file path exists before writing to it
# if !isfile(filepath)
# decisionlist = [responsedict]
# println("Creating file $filepath")
# open(filepath, "w") do io
# JSON3.pretty(io, decisionlist)
# end
# else
# # read the file and append new data
# decisionlist = copy(JSON3.read(filepath))
# push!(decisionlist, responsedict)
# println("Appending new data to file $filepath")
# open(filepath, "w") do io
# JSON3.pretty(io, decisionlist)
# end
# end
return responsedict[:score] return responsedict[:score]
end end
error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
@@ -1114,7 +1141,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
response = SQLexecution(executeSQL, sql) response = SQLexecution(executeSQL, sql)
if response[:success] if response[:success]
# intention = Dict(:intention=> "$(thoughtDict[:plan])") # intention = Dict(:intention=> "$(thoughtDict[:plan])")
extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql) extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql,
llmFormatName)
return (text=extracted, rawresponse=response[:result]) return (text=extracted, rawresponse=response[:result])
end end
end end
@@ -1330,7 +1358,7 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
reward::T3, isterminal::Bool reward::T3, isterminal::Bool
)::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict} )::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
keys = [:action_name, :action_input, :observation] keys = [:plan, :action_name, :action_input, :observation]
# latestKeys = [] # latestKeys = []
currentstate_latestKey, currentstate_latestIndice = currentstate_latestKey, currentstate_latestIndice =

21
system_prompt_template.jl
View File

@@ -6,28 +6,35 @@ Default system message template:
</Your role> </Your role>
<Situation> <Situation>
- Describe the current situation - Describe the current situation
Ex. The world use enormous energy from non-sustainable sources. This leads to climate change.
</Situation> </Situation>
<Your vision> <Your vision>
- state your vision of how the situation will evolve, what would you want the situation to evolve into - state your vision of how the situation will evolve, what would you want the situation to evolve into
Ex. To be the leading innovator in sustainable technology by 2030, transforming global energy systems.
</Your vision> </Your vision>
<Your mission> <Your mission>
- state the goal - state the goalp
Ex. Empowering communities through clean energy solutions to create a sustainable future.
</Your mission> </Your mission>
<Your mission's objective includes> <Your mission's objective includes>
- Break the goal into smaller steps - Specific, measurable, and time-bound goals that directly support the mission.
Ex. Launch 50 solar-powered water purification systems in 3 regions by 2025.
</Your mission's objective includes> </Your mission's objective includes>
<Your responsibility includes> <Your responsibility includes>
- state the mini goals that fall under your responsibility - state the mini goals that fall under your responsibility
</Your responsibility includes> </Your responsibility includes>
<Your responsibility does NOT includes> <Your responsibility does NOT includes>
- -
</Your responsibility does NOT includes> </Your responsibility does NOT includes>
<At each round of conversation, you will be given the following information> <At each round of conversation, you will be given the following information>
</At each round of conversation, you will be given the following information>
<You must follow the following guidelines>
- -
</You must follow the following guidelines> </At each round of conversation, you will be given the following information>
<You must follow the following policy>
-
</You must follow the following policy>
<You should follow the following guidelines>
-
</You should follow the following guidelines>
<You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input> <You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
Comprehension: State your comprehension about the current situation. Comprehension: State your comprehension about the current situation.
Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.

2
test/runtests.jl
View File

@@ -227,7 +227,7 @@ end
# query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?") # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?")
# query = "How many French wines from Yiem store under 100 dollars do you have?" # query = "How many French wines from Yiem store under 100 dollars do you have?"
# query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40" # query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40"
query = "from Yiem retailer, red wine from Burgundy, France. Merlot varietal. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5" query = "from Yiem retailer, red wine from France. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5"
# query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza" # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza"
# query = "wine_type: white, country: Austria, food to be served with wine: pork" # query = "wine_type: white, country: Austria, food to be served with wine: pork"
# query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza" # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"