ton/SQLLLM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat lamaiin
2025-05-14 21:21:30 +07:00
parent aeda7e0baf
commit 8c5b1b6938
3 changed files with 83 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
src/interface.jl
View File

@@ -310,14 +310,14 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
toollist = ["TABLEINFO", "RUNSQL"]
if responsedict[:action_name] ∉ toollist
errornote = "Your previous attempt has action_name that is not in the tool list"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_name]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
for i in toollist
if occursin(i, responsedict[:action_input])
errornote = "Your previous attempt has action_name in action_input which is not allowed"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_input]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
end
@@ -330,39 +330,38 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
end
end
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict))
# store for later training
responsedict[:thoughthistory] = state[:thoughtHistory]
responsedict[:system] = systemmsg
responsedict[:prompt] = prompt
responsedict[:context] = context
responsedict[:think] = think
state[:decisionMaker] = responsedict
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict))
# read sessionId
sessionid = JSON3.read("/appfolder/app/sessionid.json")
# save to filename ./log/decisionlog.txt
println("saving SQLLLM decisionMaker() to disk")
filename = "agent_decision_log_$(sessionid[:id]).json"
filepath = "/appfolder/app/log/$filename"
# check whether there is a file path exists before writing to it
if !isfile(filepath)
decisionlist = [responsedict]
println("Creating file $filepath")
open(filepath, "a") do io
JSON3.pretty(io, decisionlist)
end
else
# read the file and append new data
decisionlist = copy(JSON3.read(filepath))
push!(decisionlist, responsedict)
println("Appending new data to file $filepath")
open(filepath, "w") do io
JSON3.pretty(io, decisionlist)
end
end
# # read sessionId
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
# # save to filename ./log/decisionlog.txt
# println("saving SQLLLM decisionMaker() to disk")
# filename = "agent_decision_log_$(sessionid[:id]).json"
# filepath = "/appfolder/app/log/$filename"
# # check whether there is a file path exists before writing to it
# if !isfile(filepath)
# decisionlist = [responsedict]
# println("Creating file $filepath")
# open(filepath, "w") do io
# JSON3.pretty(io, decisionlist)
# end
# else
# # read the file and append new data
# decisionlist = copy(JSON3.read(filepath))
# push!(decisionlist, responsedict)
# println("Appending new data to file $filepath")
# open(filepath, "w") do io
# JSON3.pretty(io, decisionlist)
# end
# end
return responsedict
end
@@ -631,8 +630,6 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
Definitions:
"question" is the user's question
"understanding" is agent's understanding about the current situation
"reasoning" is agent's step-by-step reasoning about the current situation
"plan" is agent's plan to complete the task from the current situation
"action_name" is the name of the action taken, which can be one of the following functions:
- RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
@@ -642,7 +639,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
"observation" is result of the preceding immediate action
At each round of conversation, you will be given the following information:
Trajectory: ...
Trajectory: A history of how you worked on the question chronologically.
You must follow the following guidelines:
- When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
@@ -655,7 +652,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
- Do not generate additional thoughts or actions.
2) Answer_evaluation:
- Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No"
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no"
Bad example (The observation didn't answers the question):
question: Find cars with 4 wheels.
observation: There are an apple in the table.
@@ -670,7 +667,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
- 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question
- 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
- 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
5) Suggestion: what are the possible reason of this outcome, what can you learn from it and what suggestion can made?
5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
You should only respond in format as described below:
Trajectory_evaluation: ...
@@ -748,9 +745,9 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
if accepted_as_answer ∉ ["Yes", "No"]
if accepted_as_answer ∉ ["yes", "no"]
errornote = "Your previous attempt's accepted_as_answer has wrong format"
println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:accepted_as_answer]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
@@ -761,7 +758,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
state[:suggestion] = responsedict[:suggestion]
# mark as terminal state when the answer is achieved
if accepted_as_answer == "Yes"
if accepted_as_answer ∈ ["Yes", "yes"]
# mark the state as terminal state because the evaluation say so.
state[:isterminal] = true
@@ -770,10 +767,40 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
state[:reward] = responsedict[:score]
end
responsedict[:think] = think
println("\nEvaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict))
# # store for later training
# responsedict[:thoughthistory] = state[:thoughtHistory]
# responsedict[:system] = systemmsg
# responsedict[:usermsg] = usermsg
# responsedict[:prompt] = prompt
# responsedict[:context] = context
# responsedict[:think] = think
# # read sessionId
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
# # save to filename ./log/decisionlog.txt
# println("saving SQLLLM evaluator() to disk")
# filename = "agent_evaluator_log_$(sessionid[:id]).json"
# filepath = "/appfolder/app/log/$filename"
# # check whether there is a file path exists before writing to it
# if !isfile(filepath)
# decisionlist = [responsedict]
# println("Creating file $filepath")
# open(filepath, "w") do io
# JSON3.pretty(io, decisionlist)
# end
# else
# # read the file and append new data
# decisionlist = copy(JSON3.read(filepath))
# push!(decisionlist, responsedict)
# println("Appending new data to file $filepath")
# open(filepath, "w") do io
# JSON3.pretty(io, decisionlist)
# end
# end
return responsedict[:score]
end
error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
@@ -1114,7 +1141,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
response = SQLexecution(executeSQL, sql)
if response[:success]
# intention = Dict(:intention=> "$(thoughtDict[:plan])")
extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql)
extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql,
llmFormatName)
return (text=extracted, rawresponse=response[:result])
end
end
@@ -1330,7 +1358,7 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
reward::T3, isterminal::Bool
)::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
keys = [:action_name, :action_input, :observation]
keys = [:plan, :action_name, :action_input, :observation]
# latestKeys = []
currentstate_latestKey, currentstate_latestIndice =

19
system_prompt_template.jl
View File

@@ -6,15 +6,19 @@ Default system message template:
</Your role>
<Situation>
- Describe the current situation
Ex. The world use enormous energy from non-sustainable sources. This leads to climate change.
</Situation>
<Your vision>
- state your vision of how the situation will evolve, what would you want the situation to evolve into
Ex. To be the leading innovator in sustainable technology by 2030, transforming global energy systems.
</Your vision>
<Your mission>
- state the goal
- state the goalp
Ex. Empowering communities through clean energy solutions to create a sustainable future.
</Your mission>
<Your mission's objective includes>
- Break the goal into smaller steps
- Specific, measurable, and time-bound goals that directly support the mission.
Ex. Launch 50 solar-powered water purification systems in 3 regions by 2025.
</Your mission's objective includes>
<Your responsibility includes>
- state the mini goals that fall under your responsibility
@@ -23,11 +27,14 @@ Default system message template:
-
</Your responsibility does NOT includes>
<At each round of conversation, you will be given the following information>
</At each round of conversation, you will be given the following information>
<You must follow the following guidelines>
-
</You must follow the following guidelines>
</At each round of conversation, you will be given the following information>
<You must follow the following policy>
-
</You must follow the following policy>
<You should follow the following guidelines>
-
</You should follow the following guidelines>
<You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
Comprehension: State your comprehension about the current situation.
Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.

2
test/runtests.jl
View File

@@ -227,7 +227,7 @@ end
# query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?")
# query = "How many French wines from Yiem store under 100 dollars do you have?"
# query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40"
query = "from Yiem retailer, red wine from Burgundy, France. Merlot varietal. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5"
query = "from Yiem retailer, red wine from France. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5"
# query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza"
# query = "wine_type: white, country: Austria, food to be served with wine: pork"
# query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"