update
This commit is contained in:
narawat lamaiin
+68
-40
@@ -310,14 +310,14 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
|
||||
toollist = ["TABLEINFO", "RUNSQL"]
|
||||
if responsedict[:action_name] ∉ toollist
|
||||
errornote = "Your previous attempt has action_name that is not in the tool list"
|
||||
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_name]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
end
|
||||
|
||||
for i in toollist
|
||||
if occursin(i, responsedict[:action_input])
|
||||
errornote = "Your previous attempt has action_name in action_input which is not allowed"
|
||||
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_input]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
end
|
||||
end
|
||||
@@ -330,39 +330,38 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
|
||||
end
|
||||
end
|
||||
|
||||
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
pprintln(Dict(responsedict))
|
||||
|
||||
# store for later training
|
||||
responsedict[:thoughthistory] = state[:thoughtHistory]
|
||||
responsedict[:system] = systemmsg
|
||||
responsedict[:prompt] = prompt
|
||||
responsedict[:context] = context
|
||||
responsedict[:think] = think
|
||||
state[:decisionMaker] = responsedict
|
||||
|
||||
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
pprintln(Dict(responsedict))
|
||||
# read sessionId
|
||||
sessionid = JSON3.read("/appfolder/app/sessionid.json")
|
||||
|
||||
# save to filename ./log/decisionlog.txt
|
||||
println("saving SQLLLM decisionMaker() to disk")
|
||||
filename = "agent_decision_log_$(sessionid[:id]).json"
|
||||
filepath = "/appfolder/app/log/$filename"
|
||||
# check whether there is a file path exists before writing to it
|
||||
if !isfile(filepath)
|
||||
decisionlist = [responsedict]
|
||||
println("Creating file $filepath")
|
||||
open(filepath, "a") do io
|
||||
JSON3.pretty(io, decisionlist)
|
||||
end
|
||||
else
|
||||
# read the file and append new data
|
||||
decisionlist = copy(JSON3.read(filepath))
|
||||
push!(decisionlist, responsedict)
|
||||
println("Appending new data to file $filepath")
|
||||
open(filepath, "w") do io
|
||||
JSON3.pretty(io, decisionlist)
|
||||
end
|
||||
end
|
||||
# # read sessionId
|
||||
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
|
||||
# # save to filename ./log/decisionlog.txt
|
||||
# println("saving SQLLLM decisionMaker() to disk")
|
||||
# filename = "agent_decision_log_$(sessionid[:id]).json"
|
||||
# filepath = "/appfolder/app/log/$filename"
|
||||
# # check whether there is a file path exists before writing to it
|
||||
# if !isfile(filepath)
|
||||
# decisionlist = [responsedict]
|
||||
# println("Creating file $filepath")
|
||||
# open(filepath, "w") do io
|
||||
# JSON3.pretty(io, decisionlist)
|
||||
# end
|
||||
# else
|
||||
# # read the file and append new data
|
||||
# decisionlist = copy(JSON3.read(filepath))
|
||||
# push!(decisionlist, responsedict)
|
||||
# println("Appending new data to file $filepath")
|
||||
# open(filepath, "w") do io
|
||||
# JSON3.pretty(io, decisionlist)
|
||||
# end
|
||||
# end
|
||||
|
||||
return responsedict
|
||||
end
|
||||
@@ -631,8 +630,6 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
|
||||
Definitions:
|
||||
"question" is the user's question
|
||||
"understanding" is agent's understanding about the current situation
|
||||
"reasoning" is agent's step-by-step reasoning about the current situation
|
||||
"plan" is agent's plan to complete the task from the current situation
|
||||
"action_name" is the name of the action taken, which can be one of the following functions:
|
||||
- RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
|
||||
@@ -642,7 +639,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
"observation" is result of the preceding immediate action
|
||||
|
||||
At each round of conversation, you will be given the following information:
|
||||
Trajectory: ...
|
||||
Trajectory: A history of how you worked on the question chronologically.
|
||||
|
||||
You must follow the following guidelines:
|
||||
- When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
|
||||
@@ -655,7 +652,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
- Do not generate additional thoughts or actions.
|
||||
2) Answer_evaluation:
|
||||
- Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
|
||||
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No"
|
||||
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no"
|
||||
Bad example (The observation didn't answers the question):
|
||||
question: Find cars with 4 wheels.
|
||||
observation: There are an apple in the table.
|
||||
@@ -670,7 +667,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
- 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question
|
||||
- 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
|
||||
- 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
|
||||
5) Suggestion: what are the possible reason of this outcome, what can you learn from it and what suggestion can made?
|
||||
5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
|
||||
|
||||
You should only respond in format as described below:
|
||||
Trajectory_evaluation: ...
|
||||
@@ -748,9 +745,9 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
|
||||
accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
|
||||
|
||||
if accepted_as_answer ∉ ["Yes", "No"]
|
||||
if accepted_as_answer ∉ ["yes", "no"]
|
||||
errornote = "Your previous attempt's accepted_as_answer has wrong format"
|
||||
println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:accepted_as_answer]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
end
|
||||
|
||||
@@ -761,7 +758,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
state[:suggestion] = responsedict[:suggestion]
|
||||
|
||||
# mark as terminal state when the answer is achieved
|
||||
if accepted_as_answer == "Yes"
|
||||
if accepted_as_answer ∈ ["Yes", "yes"]
|
||||
|
||||
# mark the state as terminal state because the evaluation say so.
|
||||
state[:isterminal] = true
|
||||
@@ -770,10 +767,40 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
|
||||
state[:reward] = responsedict[:score]
|
||||
end
|
||||
|
||||
responsedict[:think] = think
|
||||
println("\nEvaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
pprintln(Dict(responsedict))
|
||||
|
||||
# # store for later training
|
||||
# responsedict[:thoughthistory] = state[:thoughtHistory]
|
||||
# responsedict[:system] = systemmsg
|
||||
# responsedict[:usermsg] = usermsg
|
||||
# responsedict[:prompt] = prompt
|
||||
# responsedict[:context] = context
|
||||
# responsedict[:think] = think
|
||||
|
||||
# # read sessionId
|
||||
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
|
||||
# # save to filename ./log/decisionlog.txt
|
||||
# println("saving SQLLLM evaluator() to disk")
|
||||
# filename = "agent_evaluator_log_$(sessionid[:id]).json"
|
||||
# filepath = "/appfolder/app/log/$filename"
|
||||
# # check whether there is a file path exists before writing to it
|
||||
# if !isfile(filepath)
|
||||
# decisionlist = [responsedict]
|
||||
# println("Creating file $filepath")
|
||||
# open(filepath, "w") do io
|
||||
# JSON3.pretty(io, decisionlist)
|
||||
# end
|
||||
# else
|
||||
# # read the file and append new data
|
||||
# decisionlist = copy(JSON3.read(filepath))
|
||||
# push!(decisionlist, responsedict)
|
||||
# println("Appending new data to file $filepath")
|
||||
# open(filepath, "w") do io
|
||||
# JSON3.pretty(io, decisionlist)
|
||||
# end
|
||||
# end
|
||||
|
||||
return responsedict[:score]
|
||||
end
|
||||
error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
|
||||
@@ -1114,7 +1141,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
response = SQLexecution(executeSQL, sql)
|
||||
if response[:success]
|
||||
# intention = Dict(:intention=> "$(thoughtDict[:plan])")
|
||||
extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql)
|
||||
extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql,
|
||||
llmFormatName)
|
||||
return (text=extracted, rawresponse=response[:result])
|
||||
end
|
||||
end
|
||||
@@ -1330,7 +1358,7 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
|
||||
reward::T3, isterminal::Bool
|
||||
)::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
|
||||
|
||||
keys = [:action_name, :action_input, :observation]
|
||||
keys = [:plan, :action_name, :action_input, :observation]
|
||||
# latestKeys = []
|
||||
|
||||
currentstate_latestKey, currentstate_latestIndice =
|
||||
|
||||
Reference in New Issue
Block a user