diff --git a/src/interface.jl b/src/interface.jl index d7ac619..8a542d0 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -310,14 +310,14 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function toollist = ["TABLEINFO", "RUNSQL"] if responsedict[:action_name] ∉ toollist errornote = "Your previous attempt has action_name that is not in the tool list" - println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_name]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end for i in toollist if occursin(i, responsedict[:action_input]) errornote = "Your previous attempt has action_name in action_input which is not allowed" - println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:action_input]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end end @@ -330,39 +330,38 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function end end + println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(Dict(responsedict)) + # store for later training responsedict[:thoughthistory] = state[:thoughtHistory] responsedict[:system] = systemmsg responsedict[:prompt] = prompt responsedict[:context] = context responsedict[:think] = think - state[:decisionMaker] = responsedict - println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - pprintln(Dict(responsedict)) - # read sessionId - sessionid = JSON3.read("/appfolder/app/sessionid.json") - - # save to filename ./log/decisionlog.txt - println("saving SQLLLM decisionMaker() to disk") - filename = "agent_decision_log_$(sessionid[:id]).json" - filepath = "/appfolder/app/log/$filename" - # check whether there is a file path exists before writing to it - if !isfile(filepath) - decisionlist = [responsedict] - println("Creating file $filepath") - open(filepath, "a") do io - JSON3.pretty(io, decisionlist) - end - else - # read the file and append new data - decisionlist = copy(JSON3.read(filepath)) - push!(decisionlist, responsedict) - println("Appending new data to file $filepath") - open(filepath, "w") do io - JSON3.pretty(io, decisionlist) - end - end + # # read sessionId + # sessionid = JSON3.read("/appfolder/app/sessionid.json") + # # save to filename ./log/decisionlog.txt + # println("saving SQLLLM decisionMaker() to disk") + # filename = "agent_decision_log_$(sessionid[:id]).json" + # filepath = "/appfolder/app/log/$filename" + # # check whether there is a file path exists before writing to it + # if !isfile(filepath) + # decisionlist = [responsedict] + # println("Creating file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # else + # # read the file and append new data + # decisionlist = copy(JSON3.read(filepath)) + # push!(decisionlist, responsedict) + # println("Appending new data to file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # end return responsedict end @@ -631,8 +630,6 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll Definitions: "question" is the user's question - "understanding" is agent's understanding about the current situation - "reasoning" is agent's step-by-step reasoning about the current situation "plan" is agent's plan to complete the task from the current situation "action_name" is the name of the action taken, which can be one of the following functions: - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. @@ -642,7 +639,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll "observation" is result of the preceding immediate action At each round of conversation, you will be given the following information: - Trajectory: ... + Trajectory: A history of how you worked on the question chronologically. You must follow the following guidelines: - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. @@ -655,7 +652,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll - Do not generate additional thoughts or actions. 2) Answer_evaluation: - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question - 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No" + 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no" Bad example (The observation didn't answers the question): question: Find cars with 4 wheels. observation: There are an apple in the table. @@ -670,7 +667,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. - 5) Suggestion: what are the possible reason of this outcome, what can you learn from it and what suggestion can made? + 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made? You should only respond in format as described below: Trajectory_evaluation: ... @@ -748,9 +745,9 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll accepted_as_answer::AbstractString = responsedict[:accepted_as_answer] - if accepted_as_answer ∉ ["Yes", "No"] + if accepted_as_answer ∉ ["yes", "no"] errornote = "Your previous attempt's accepted_as_answer has wrong format" - println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --> $(responsedict[:accepted_as_answer]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end @@ -761,7 +758,7 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll state[:suggestion] = responsedict[:suggestion] # mark as terminal state when the answer is achieved - if accepted_as_answer == "Yes" + if accepted_as_answer ∈ ["Yes", "yes"] # mark the state as terminal state because the evaluation say so. state[:isterminal] = true @@ -770,10 +767,40 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll state[:reward] = responsedict[:score] end - responsedict[:think] = think - println("\nEvaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") pprintln(Dict(responsedict)) + # # store for later training + # responsedict[:thoughthistory] = state[:thoughtHistory] + # responsedict[:system] = systemmsg + # responsedict[:usermsg] = usermsg + # responsedict[:prompt] = prompt + # responsedict[:context] = context + # responsedict[:think] = think + + # # read sessionId + # sessionid = JSON3.read("/appfolder/app/sessionid.json") + # # save to filename ./log/decisionlog.txt + # println("saving SQLLLM evaluator() to disk") + # filename = "agent_evaluator_log_$(sessionid[:id]).json" + # filepath = "/appfolder/app/log/$filename" + # # check whether there is a file path exists before writing to it + # if !isfile(filepath) + # decisionlist = [responsedict] + # println("Creating file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # else + # # read the file and append new data + # decisionlist = copy(JSON3.read(filepath)) + # push!(decisionlist, responsedict) + # println("Appending new data to file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # end + return responsedict[:score] end error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") @@ -1114,7 +1141,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; response = SQLexecution(executeSQL, sql) if response[:success] # intention = Dict(:intention=> "$(thoughtDict[:plan])") - extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql) + extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql, + llmFormatName) return (text=extracted, rawresponse=response[:result]) end end @@ -1330,7 +1358,7 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response:: reward::T3, isterminal::Bool )::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict} - keys = [:action_name, :action_input, :observation] + keys = [:plan, :action_name, :action_input, :observation] # latestKeys = [] currentstate_latestKey, currentstate_latestIndice = diff --git a/system_prompt_template.jl b/system_prompt_template.jl index 158c943..3b88fd2 100644 --- a/system_prompt_template.jl +++ b/system_prompt_template.jl @@ -6,28 +6,35 @@ Default system message template: - Describe the current situation + Ex. The world use enormous energy from non-sustainable sources. This leads to climate change. - state your vision of how the situation will evolve, what would you want the situation to evolve into + Ex. To be the leading innovator in sustainable technology by 2030, transforming global energy systems. - - state the goal + - state the goalp + Ex. Empowering communities through clean energy solutions to create a sustainable future. - - Break the goal into smaller steps + - Specific, measurable, and time-bound goals that directly support the mission. + Ex. Launch 50 solar-powered water purification systems in 3 regions by 2025. - state the mini goals that fall under your responsibility - - + - - - - - - + + + - + + + - + Comprehension: State your comprehension about the current situation. Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. diff --git a/test/runtests.jl b/test/runtests.jl index 6658b52..7d8849b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -227,7 +227,7 @@ end # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?") # query = "How many French wines from Yiem store under 100 dollars do you have?" # query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40" -query = "from Yiem retailer, red wine from Burgundy, France. Merlot varietal. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5" +query = "from Yiem retailer, red wine from France. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5" # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza" # query = "wine_type: white, country: Austria, food to be served with wine: pork" # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"