diff --git a/Manifest.toml b/Manifest.toml index 74eb80b..1a41a0e 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "6e0efa362e5748de9ce219e0376be2f93d464376" +project_hash = "ec4f3941a75715b7ba32ddf816f49fe57098c82d" [[deps.Accessors]] deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] @@ -270,17 +270,17 @@ version = "1.11.0" [[deps.GeneralUtils]] deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"] -git-tree-sha1 = "f1bad7621e6ac2d235adc8c593b0aff87bd1d93e" +git-tree-sha1 = "b172f75aa622507027cd269af2cc5f335e9821eb" repo-rev = "main" repo-url = "https://git.yiem.cc/ton/GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -version = "0.4.2" +version = "0.4.4" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"] -git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55" +git-tree-sha1 = "e718a35dd7386ccd6bed64a1d84d661972404b99" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "2.5.0" +version = "2.5.1" [[deps.HashArrayMappedTries]] git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" @@ -752,7 +752,7 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" [[deps.SQLLLM]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "HTTP", "JSON", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"] path = "." uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3" version = "0.2.4" diff --git a/Project.toml b/Project.toml index 8522217..a8742ca 100644 --- a/Project.toml +++ b/Project.toml @@ -24,5 +24,5 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] Dates = "1.11.0" -GeneralUtils = "0.4.2" +GeneralUtils = "0.4.4" JSON = "1.6.1" diff --git a/README.md b/README.md index e69de29..e2ee34b 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/interface.jl b/src/interface.jl index d357542..b330672 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -34,7 +34,7 @@ julia> state = Dict( "reward" => 0, "evaluation" => "None", "accepted_as_answer" => "No", - "thoughtHistory" => OrderedDict{String, Any}("question" => "How many wines do you have that can be paired with lamb?"), + "action_history" => OrderedDict{String, Any}("question" => "How many wines do you have that can be paired with lamb?"), "evaluationscore" => 0, "suggestion" => "None" ) @@ -102,141 +102,70 @@ Dict( # Signature """ -function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function, llmFormatName::String +function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName::String ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10 )::Dict{String, Any} where {T1<:AbstractDict, T2<:Function} - systemmsg = - """ - You are a helpful assistant that find the data from a database to satisfy the user's question. - You are working under your mentor supervision and you are also eager to improve your helpfulness. + requiredKeys = ["plan", "action_name", "action_input"] + errornote = "" + # provide similar sql only for the first attempt + sql, distance = querySQLVectorDBF(state["question"]) + similarSQL_ = sql !== nothing ? sql : "None" + context = + """ + + $(GeneralUtils.dict_to_string_html(state["context"])) + + $similarSQL_ + + + $(GeneralUtils.dict_to_string_html(state["action_history"])) + + + $errornote + + + """ - For your information: - - Observation: Result of the immediately preceding action - - At each round of conversation, you will be given the following information: - context: additional information about the current situation - - You must follow the following guidelines: - - Keep SQL queries focused only on the provided information. - - You should follow the following guidelines: - - Do not create any table in the database - - A junction table can be used to link tables together. Another use case is for filtering data. - - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. - - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. - - If there is no search result from the database, remove the restrictive criteria until a search result is available, and proceed from there. - - You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input: - 1) plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. - 2) action_name: (Typically corresponds to the execution of the first step in your plan) - Can be one of the following function names: - - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. - For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. - Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. - 3) action_input: Input to the action - - You should only respond in JSON format as described below: - { - "plan": "...", - "action_name": "...", - "action_input": "..." - } - - Let's begin! - """ - requiredKeys = [:plan, :action_name, :action_input] - workprogress = "" - for (k, v) in state["thoughtHistory"] - if k ∉ ["question"] - workprogress *= "$k: $v\n" + # add context to text of the latest message (in the front). + # use for loop because in openai format, each msg may contain both text and image. + + for d in state["chathistory"][end]["content"] + if d["type"] == "text" + d["text"] = context * d["text"] + break end end - + response = nothing # store for show when error msg show up - errornote = "N/A" - - # provide similar sql only for the first attempt -similarSQL_ = "None" - if length(state["thoughtHistory"]) == 1 - sql, distance = querySQLVectorDBF(state["thoughtHistory"]["question"]) - similarSQL_ = sql !== nothing ? sql : "None" - end - for attempt in 1:maxattempt + msg = Dict( + "model" => "gemma-4-E4B-it-UD-Q4_K_XL", + "messages" => state["chathistory"], + "temperature" => 0.7 + ) + response = text2textInstructLLM("random_id", msg) + response = GeneralUtils.clean_json_response(response) - # QandA = generatequestion(state, context, text2textInstructLLM, llmFormatName; similarSQL=similarSQL_) - - context = - """ - - This is schema of tables in the database: - $(additionalinfo[:tablelist]) - - The closest known SQL for this question is: - $similarSQL_ - - This is the query result when executing the most_relevant_SQL against a database. You can use this to see how the data are stored. - winery: Chateau Montelena, wine_name: The Montelena Estate Cabernet Sauvignon, wine_id: 97264f71-007c-4cce-a3fe-2cc88fba4d05, vintage: 2017, region: Napa Valley, country: United States, wine_type: red, grape: Cabernet Sauvignon, serving_temperature: 15 to 18 Celsius, sweetness: 1, intensity: 5, tannin: 4, acidity: 4, tasting_notes: oak, vanilla, tobacco, blackberry, plum, black cherry, leather, earthy, smoke, price: 19.95, currency: USD - - your work progress so far: - $workprogress - - This is your mentor's suggestion for the immediately preceding action and observation -$(state["suggestion"]) - - P.S. $errornote - - """ - - unformatPrompt = - [ - Dict(:name => "system", :text => systemmsg), - Dict("name" => "user", "text" => state["thoughtHistory"]["question"]) - ] - - # put in model format - prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName) - # add info - prompt = prompt * context - response = text2textInstructLLM(prompt) - response = GeneralUtils.deFormatLLMtext(response, llmFormatName) think, response = GeneralUtils.extractthink(response) - - # if occursin("NULL", response) - # errornote = "\nYour previous attempt contain NULL. It is not allowed in your response" - # println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> \n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # continue - # end - responsedict = nothing - try - responsedict = copy(JSON.parse(response)) + try + _responsedict = JSON.parse(response) + responsedict = GeneralUtils.dictify(_responsedict, keytype=String) catch - println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end # check whether all answer's key points are in responsedict - _responsedictKey = keys(responsedict) - responsedictKey = [i for i in _responsedictKey] # convert into a list - is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] - - if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) - errornote = "Your previous attempt has more key points than answer's required key points." - println("\nERROR YiemAgent generatechat() $errornote --(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif !all(is_requiredKeys_in_responsedictKey) - zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) - missingkeys = [requiredKeys[i] for i in zeroind] - errornote = "$missingkeys are missing from your previous response" - println("\nERROR YiemAgent generatechat() $errornote --(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys) + if !ispass + errornote = errormsg + println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") continue end - delete!(responsedict, "observation") - # remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String}) if occursin("```", responsedict["action_input"]) sql = GeneralUtils.extract_triple_backtick_text(responsedict["action_input"])[1] @@ -249,7 +178,7 @@ $(state["suggestion"]) responsedict["action_input"] = sql end - toollist = ["TABLEINFO", "RUNSQL"] + toollist = ["RUNSQL"] if responsedict["action_name"] ∉ toollist errornote = "Your previous attempt has action_name that is not in the tool list" println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["action_name"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") @@ -263,284 +192,16 @@ $(state["suggestion"]) continue end end - - # for i ∈ Symbol.(dictkey) - # if length(JSON3.write(responsedict[i])) == 0 - # errornote = "Your previous attempt has empty value for $i" - # println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # continue - # end - # end - + println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - pprintln(Dict(responsedict)) - - # store for later training -responsedict["thoughthistory"] = state["thoughtHistory"] - responsedict["system"] = systemmsg - responsedict["prompt"] = prompt - responsedict["context"] = context - responsedict["think"] = think - - # # read sessionId - # sessionid = JSON.parse("/appfolder/app/sessionid.json") - # # save to filename ./log/decisionlog.txt - # println("saving SQLLLM decisionMaker() to disk") - # filename = "agent_decision_log_$(sessionid[:id]).json" - # filepath = "/appfolder/app/log/$filename" - # # check whether there is a file path exists before writing to it - # if !isfile(filepath) - # decisionlist = [responsedict] - # println("Creating file $filepath") - # open(filepath, "w") do io - # JSON3.pretty(io, decisionlist) - # end - # else - # # read the file and append new data - # decisionlist = copy(JSON.parse(filepath)) - # push!(decisionlist, responsedict) - # println("Appending new data to file $filepath") - # open(filepath, "w") do io - # JSON3.pretty(io, decisionlist) - # end - # end - + pprintln(responsedict) + println("---") + return responsedict end error("SQLLLM DecisionMaker() failed to generate a thought \n", response) end -# function decisionMaker(state::T1, context, text2textInstructLLM::Function, llmFormatName::String -# ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10 -# )::Dict{String, Any} where {T1<:AbstractDict, T2<:Function} - -# # lessonDict = -# # if isfile("lesson.json") -# # lessonDict = copy(JSON.parse("lesson.json")) -# # else -# # lessonDict = nothing -# # end - -# # lessonDict = nothing - -# # lesson = -# # if lessonDict === nothing -# # "" -# # else -# # """ -# # You have attempted to help the user before and failed, either because your reasoning for the -# # recommendation was incorrect or your response did not exactly match the user expectation. -# # The following lesson(s) give a plan to avoid failing to help the user in the same way you -# # did previously. Use them to improve your strategy to help the user. - -# # Here are some lessons in JSON format: -# # $(JSON3.write(lessonDict)) - -# # When providing the thought and action for the current trial, that into account these failed -# # trajectories and make sure not to repeat the same mistakes and incorrect answers. -# # """ -# # end - -# systemmsg = -# """ -# You are a helpful assistant that find the data from a database to satisfy the user's query. -# You are also eager to improve your helpfulness. - -# For your information: -# - Observation: Result of the immediately preceding action - -# At each round of conversation, the user will give you the following: -# User Query: ... -# Example: ... -# Your Q&A: ... -# Your work progress: ... -# Evaluation: Evaluation of the immediately preceding action and observation -# Suggestion: Suggestion for the immediately preceding action and observation - -# You must follow the following guidelines: -# - Keep SQL queries focused only on the provided information. - -# You should follow the following guidelines: -# - Do not create any table in the database -# - A junction table can be used to link tables together. Another use case is for filtering data. -# - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. -# - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query". -# - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. - -# You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input: -# Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. -# Action_name: (Typically corresponds to the execution of the first step in your plan) -# Can be one of the following function names: -# - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. -# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. -# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. -# 4) Action_input: Input to the action - -# You should only respond in format as described below: -# Plan: ... -# Action_name: ... -# Action_input: ... - -# Let's begin! -# """ - -# workprogress = "" -# for (k, v) in state[:thoughtHistory] -# if k ∉ [:question] -# workprogress *= "$k: $v\n" -# end -# end - -# response = nothing # store for show when error msg show up -# errornote = "N/A" - -# # provide similar sql only for the first attempt -# similarSQL_ = "None" -# if length(state[:thoughtHistory]) == 1 -# sql, distance = querySQLVectorDBF(state[:thoughtHistory][:question]) -# similarSQL_ = sql !== nothing ? sql : "None" -# end - -# header = ["Plan:", "Action_name:", "Action_input:"] -# dictkey = ["plan", "action_name", "action_input"] - -# llmkwargs=Dict( -# :num_ctx => 32768, -# :temperature => 0.5, -# ) - -# for attempt in 1:maxattempt - -# QandA = generatequestion(state, context, text2textInstructLLM, llmFormatName; similarSQL=similarSQL_) - -# usermsg = -# """ -# $(context[:tablelist]) -# User query: $(state[:thoughtHistory][:question]) -# Example: $similarSQL_ -# Your Q&A: $QandA -# Your work progress: $workprogress -# Evaluation: $(state[:evaluation]) -# Suggestion: $(state[:suggestion]) -# P.S. $errornote -# """ - -# _prompt = -# [ -# Dict(:name=> "system", :text=> systemmsg), -# Dict(:name=> "user", :text=> usermsg) -# ] - -# # put in model format -# prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName) -# response = text2textInstructLLM(prompt; llmkwargs=llmkwargs) -# response = GeneralUtils.deFormatLLMtext(response, llmFormatName) -# think, response = GeneralUtils.extractthink(response) - -# # LLM tends to generate observation given that it is in the input -# response = -# if occursin("observation:", response) -# string(split(response, "observation:")[1]) -# elseif occursin("Observation:", response) -# string(split(response, "Observation:")[1]) -# elseif occursin("observation_", response) -# string(split(response, "observation_")[1]) -# elseif occursin("Observation_", response) -# string(split(response, "Observation_")[1]) -# else -# response -# end - -# # sometime LLM output something like **Comprehension**: which is not expected -# response = replace(response, "**"=>"") -# response = replace(response, "***"=>"") - -# # some time LLM output Plan_1: so we need to detect and replace topic numbering -# regex = r"_[0-1000]+:" -# matches = collect(eachmatch(regex, response)) -# for m in matches -# response = replace(response, string(m.match)=>":") -# end - -# if occursin("NULL", response) -# errornote = "\nYour previous attempt was NULL. This is not allowed" -# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# # # detect if there are more than 1 key per categories -# # wordcount = GeneralUtils.countGivenWords(response, header) -# # duplicateKeywordFlag = false -# # for (i, v) in enumerate(wordcount) -# # keyword = header[i] -# # keywordNumber = v -# # if keywordNumber > 1 -# # errornote = "\nSQL query has duplicated keyword, $keyword" -# # println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# # duplicateKeywordFlag = true -# # break -# # end -# # end -# # duplicateKeywordFlag == true ? continue : nothing - -# # check whether response has all header -# detected_kw = GeneralUtils.detect_keyword(header, response) -# if 0 ∈ values(detected_kw) -# errornote = "\nYour previous attempt did not have all points according to the required response format" -# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# elseif sum(values(detected_kw)) > length(header) -# errornote = "\nYour previous attempt has duplicated points according to the required response format" -# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# responsedict = GeneralUtils.textToDict(response, header; -# dictKey=dictkey, symbolkey=false) - -# delete!(responsedict, :observation) - -# # remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String}) -# if occursin("```", responsedict[:action_input]) -# sql = GeneralUtils.extract_triple_backtick_text(responsedict[:action_input])[1] -# if sql[1:4] == "sql\n" -# sql = sql[5:end] -# end -# sql = split(sql, ';') # some time there are comments in the sql -# sql = sql[1] * ';' - -# responsedict[:action_input] = sql -# end - -# toollist = ["TABLEINFO", "RUNSQL"] -# if responsedict[:action_name] ∉ toollist -# errornote = "Your previous attempt has action_name that is not in the tool list" -# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# for i in toollist -# if occursin(i, responsedict[:action_input]) -# errornote = "Your previous attempt has action_name in action_input which is not allowed" -# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end -# end - -# for i ∈ Symbol.(dictkey) -# if length(JSON3.write(responsedict[i])) == 0 -# errornote = "Your previous attempt has empty value for $i" -# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end -# end - -# state[:decisionMaker] = responsedict -# return responsedict -# end -# error("SQLLLM DecisionMaker() failed to generate a thought \n", response) -# end - """ Assigns a scalar value to each new child node to be used for selec- tion and backpropagation. This value effectively quantifies the agent's progress in task completion, @@ -562,125 +223,120 @@ julia> # Signature """ -function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String; +function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::String; maxattempt=10 ) where {T1<:AbstractDict} - systemmsg = + systemmsg = """ - You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions. - - Definitions: - "question" is the user's question - "plan" is agent's plan to complete the task from the current situation - "action_name" is the name of the action taken, which can be one of the following functions: - - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. - For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. - Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. - "action_input" is the input to the action - "observation" is result of the preceding immediate action - - At each round of conversation, you will be given the following information: - trajectory: A history of how you worked on the question chronologically - evaluatee_context: The context that evaluatee use to make a decision - - You must follow the following guidelines: - - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. - - You should then respond to the user with: - 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. - - Evaluate the correctness of each section and the overall trajectory based on the given question. - - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation. - - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. - - Do not generate additional thoughts or actions. - 2) Answer_evaluation: - - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question - 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no" - Bad example (The observation didn't answers the question): - question: Find cars with 4 wheels. - observation: There are an apple in the table. - Good example (The observation answers the question): - question: Find cars with a stereo. - observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black. - 4) Score: Correctness score s where s is a single integer between 0 to 9. + + At each round of conversation, the user provides the following: + - customer question + - trajectory: A history of how an agent (you) worked on the question chronologically + + + Analyze and evaluate agent's trajectory to find solutions and the results of actions to answer the user's questions according to evaluation guidelines. + + + Fulfill the objective. + + + - When the search returns no result, it usually means 1) there is simply no data. or 2) SQL condition is not correct or 3) SQL is looking at the wrong tables. + - validate whether the SQL query makes sense before accepting it as a valid answer. + + + 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. + - Evaluate the correctness of each section and the overall trajectory based on the given question. + - Provide detailed reasoning and analysis, focusing on the latest plan, action_name, action_input, and action_result. + - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. + - Do not generate additional thoughts or actions. + 2) Answer_evaluation: + - Focus only on the matter mentioned in the question and comprehensively analyze how the latest action_input is appropriate. + 3) Accepted_as_answer: Decide whether the latest action_input is technically correct. Can be "yes" or "no" + Bad example: + question: Find cars with 4 wheels. + action_input: INSERT INTO employees + VALUES (5, 'Charlie', 'Green', '2026-06-01', 60000.00);. + Good example: + question: Find cars with a sunroof. + action_input: SELECT * FROM car_features + WHERE has_sunroof = TRUE; + 4) Score: Correctness score s where s is a single integer between 0 to 9. For example: - - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed + - 0 indicates that both the trajectory is incorrect, failed or errors and the action_result is incorrect or failed - 4 indicates that the trajectory are correct, but no results are returned. - - 5 indicates that the trajectory are correct but the observation is incorrect or failed - - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question - - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. - - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. - 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made? - - You should only respond in format as described below: - Trajectory_evaluation: ... - Answer_evaluation: ... - Accepted_as_answer: ... - Score: ... - Suggestion: ... - - Let's begin! + - 5 indicates that the trajectory are correct but the action_result is incorrect or failed + - 6 indicates that the trajectory are correct, but the action_result's content doesn't directly answer the question + - 8 indicates that both the trajectory are correct, and the action_result's content directly answers the question. + - 9 indicates a perfect perfomance. Both the trajectory are correct, and the action_result's content directly answers the question, surpassing your expectations. + 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made? + + + "trajectory_evaluation": "...", + "answer_evaluation": "...", + "accepted_as_answer": "...", + "score": "...", + "suggestion": "..." + + """ + requiredKeys = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"] + errornote = "" + usermsg = + """ + + $(state["context"]["table_schema"]) + + + $(state["chathistory"][2]["content"][1]["text"]) + + + $(GeneralUtils.dict_to_string_html(state["action_history"])) + """ - #[WORKING] add what I should think --> this will be the think for decisionMaker() - header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"] - dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"] - thoughthistory = "" - for (k, v) in state["thoughtHistory"] - thoughthistory *= "$k: $v\n" - end - - errornote = "N/A" + msg = Dict( + "model" => "gemma-4-E4B-it-UD-Q4_K_XL", + "messages" => [ + Dict( + "role" => "system", + "content" => [ + Dict("type" => "text", "text" => systemmsg), + ] + ), + Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => usermsg), + ] + ), + ], + "temperature" => 0.7 + ) + for attempt in 1:maxattempt - usermsg = - """ - - $thoughthistory - - """ - context = - """ - - - thoughtDict[:context] - - P.S. $errornote - - """ + response = text2textInstructLLM("random_id", msg) + response = GeneralUtils.clean_json_response(response) - unformatPrompt = - [ - Dict(:name => "system", :text => systemmsg), - Dict(:name => "user", :text => usermsg) - ] - # put in model format - prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName) - # add info - prompt = prompt * context - - response = text2textInstructLLM(prompt, modelsize="medium") - response = GeneralUtils.deFormatLLMtext(response, llmFormatName) + response = GeneralUtils.remove_french_accents(response) think, response = GeneralUtils.extractthink(response) - - # sometime LLM output something like **Comprehension**: which is not expected - response = replace(response, "**"=>"") - response = replace(response, "***"=>"") - - # check whether response has all header - detected_kw = GeneralUtils.detectKeywordVariation(header, response) - missingkeys = [k for (k, v) in detected_kw if v === nothing] - if !isempty(missingkeys) - errornote = "$missingkeys are missing from your previous response" - println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum([length(i) for i in values(detected_kw)]) > length(header) - errornote = "\nYour previous attempt has duplicated points according to the required response format" - println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + response = String(split(response, ", action_result")[1]) # in case LLM generate action_result key which it isn't supposed to + response = strip(response) + responsedict = nothing + try + _responsedict = JSON.parse(response) + responsedict = GeneralUtils.dictify(_responsedict, keytype=String) + catch + println("\nERROR SQLLLM evaluator() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=false) + # check whether all answer's key points are in responsedict + ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys) + if !ispass + errornote = errormsg + println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") + continue + end responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. try @@ -693,7 +349,7 @@ function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFo accepted_as_answer::AbstractString = responsedict["accepted_as_answer"] - if accepted_as_answer ∉ ["yes", "no"] + if accepted_as_answer ∉ ["Yes", "yes", "No", "no"] errornote = "Your previous attempt's accepted_as_answer has wrong format" println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["accepted_as_answer"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue @@ -715,44 +371,205 @@ function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFo state["reward"] = responsedict["score"] end - println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - pprintln(Dict(responsedict)) - - # # store for later training - # responsedict[:thoughthistory] = state[:thoughtHistory] - # responsedict[:system] = systemmsg - # responsedict[:usermsg] = usermsg - # responsedict[:prompt] = prompt - # responsedict[:context] = context - # responsedict[:think] = think - - # # read sessionId - # sessionid = JSON.parse("/appfolder/app/sessionid.json") - # # save to filename ./log/decisionlog.txt - # println("saving SQLLLM evaluator() to disk") - # filename = "agent_evaluator_log_$(sessionid[:id]).json" - # filepath = "/appfolder/app/log/$filename" - # # check whether there is a file path exists before writing to it - # if !isfile(filepath) - # decisionlist = [responsedict] - # println("Creating file $filepath") - # open(filepath, "w") do io - # JSON3.pretty(io, decisionlist) - # end - # else - # # read the file and append new data - # decisionlist = copy(JSON.parse(filepath)) - # push!(decisionlist, responsedict) - # println("Appending new data to file $filepath") - # open(filepath, "w") do io - # JSON3.pretty(io, decisionlist) - # end - # end + println("\n--- SQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(responsedict) + println("---\n") return responsedict["score"] end error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") end +# function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String; +# maxattempt=10 +# ) where {T1<:AbstractDict} + +# systemmsg = +# """ +# You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions. + +# Definitions: +# "question" is the user's question +# "plan" is agent's plan to complete the task from the current situation +# "action_name" is the name of the action taken, which can be one of the following functions: +# - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. +# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. +# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. +# "action_input" is the input to the action +# "observation" is result of the preceding immediate action + +# At each round of conversation, you will be given the following information: +# trajectory: A history of how you worked on the question chronologically +# evaluatee_context: The context that evaluatee use to make a decision + +# You must follow the following guidelines: +# - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. + +# You should then respond to the user with: +# 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. +# - Evaluate the correctness of each section and the overall trajectory based on the given question. +# - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation. +# - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. +# - Do not generate additional thoughts or actions. +# 2) Answer_evaluation: +# - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question +# 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no" +# Bad example (The observation didn't answers the question): +# question: Find cars with 4 wheels. +# observation: There are an apple in the table. +# Good example (The observation answers the question): +# question: Find cars with a stereo. +# observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black. +# 4) Score: Correctness score s where s is a single integer between 0 to 9. +# For example: +# - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed +# - 4 indicates that the trajectory are correct, but no results are returned. +# - 5 indicates that the trajectory are correct but the observation is incorrect or failed +# - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question +# - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. +# - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. +# 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made? + +# You should only respond in format as described below: +# Trajectory_evaluation: ... +# Answer_evaluation: ... +# Accepted_as_answer: ... +# Score: ... +# Suggestion: ... + +# Let's begin! +# """ +# #[WORKING] add what I should think --> this will be the think for decisionMaker() +# header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"] +# dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"] + +# action_history = "" +# for (k, v) in state["action_history"] +# action_history *= "$k: $v\n" +# end + +# errornote = "N/A" +# for attempt in 1:maxattempt +# usermsg = +# """ +# +# $action_history +# +# """ +# context = +# """ +# +# +# thoughtDict[:context] +# +# P.S. $errornote +# +# """ + +# unformatPrompt = +# [ +# Dict(:name => "system", :text => systemmsg), +# Dict(:name => "user", :text => usermsg) +# ] +# # put in model format +# prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName) +# # add info +# prompt = prompt * context + +# response = text2textInstructLLM(prompt, modelsize="medium") +# response = GeneralUtils.deFormatLLMtext(response, llmFormatName) +# think, response = GeneralUtils.extractthink(response) + +# # sometime LLM output something like **Comprehension**: which is not expected +# response = replace(response, "**"=>"") +# response = replace(response, "***"=>"") + +# # check whether response has all header +# detected_kw = GeneralUtils.detectKeywordVariation(header, response) +# missingkeys = [k for (k, v) in detected_kw if v === nothing] +# if !isempty(missingkeys) +# errornote = "$missingkeys are missing from your previous response" +# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# continue +# elseif sum([length(i) for i in values(detected_kw)]) > length(header) +# errornote = "\nYour previous attempt has duplicated points according to the required response format" +# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# continue +# end + +# responsedict = GeneralUtils.textToDict(response, header; +# dictKey=dictkey, symbolkey=false) + +# responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. +# try +# responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5 +# catch +# errornote = "Your previous attempt's score has wrong format" +# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# continue +# end + +# accepted_as_answer::AbstractString = responsedict["accepted_as_answer"] + +# if accepted_as_answer ∉ ["yes", "no"] +# errornote = "Your previous attempt's accepted_as_answer has wrong format" +# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["accepted_as_answer"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# continue +# end + +# # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension) +# state["evaluation"] = "$(responsedict["trajectory_evaluation"]) $(responsedict["answer_evaluation"])" +# state["evaluationscore"] = responsedict["score"] +# state["accepted_as_answer"] = responsedict["accepted_as_answer"] +# state["suggestion"] = responsedict["suggestion"] + +# # mark as terminal state when the answer is achieved +# if accepted_as_answer ∈ ["Yes", "yes"] + +# # mark the state as terminal state because the evaluation say so. +# state["isterminal"] = true + +# # evaluation score as reward because different answers hold different value for the user. +# state["reward"] = responsedict["score"] +# end + +# println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# pprintln(Dict(responsedict)) + +# # # store for later training +# # responsedict[:action_history] = state[:action_history] +# # responsedict[:system] = systemmsg +# # responsedict[:usermsg] = usermsg +# # responsedict[:prompt] = prompt +# # responsedict[:context] = context +# # responsedict[:think] = think + +# # # read sessionId +# # sessionid = JSON.parse("/appfolder/app/sessionid.json") +# # # save to filename ./log/decisionlog.txt +# # println("saving SQLLLM evaluator() to disk") +# # filename = "agent_evaluator_log_$(sessionid[:id]).json" +# # filepath = "/appfolder/app/log/$filename" +# # # check whether there is a file path exists before writing to it +# # if !isfile(filepath) +# # decisionlist = [responsedict] +# # println("Creating file $filepath") +# # open(filepath, "w") do io +# # JSON3.pretty(io, decisionlist) +# # end +# # else +# # # read the file and append new data +# # decisionlist = copy(JSON.parse(filepath)) +# # push!(decisionlist, responsedict) +# # println("Appending new data to file $filepath") +# # open(filepath, "w") do io +# # JSON3.pretty(io, decisionlist) +# # end +# # end + +# return responsedict["score"] +# end +# error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") +# end """ @@ -839,7 +656,7 @@ function reflector(config::T1, state::T2)::String where {T1<:AbstractDict, T2<:A usermsg = """ - $(JSON3.write(state[:thoughtHistory])) + $(JSON.json(state[:action_history])) """ _prompt = @@ -930,24 +747,20 @@ julia> state = Dict( "reward" => 0, "evaluation" => "None", "accepted_as_answer" => "No", - "thoughtHistory" => OrderedDict{String, Any}("question" => "How many wines do you have that can be paired with lamb?"), + "action_history" => OrderedDict{String, Any}("question" => "How many wines do you have that can be paired with lamb?"), "evaluationscore" => 0, "suggestion" => "None" ) ``` - -# TODO - - [] add embedding of newstate and store in newstate[:embedding] - - [PENDING] should getdata() return isterminal? -# Signature -""" + - add embedding of newstate and store in newstate[:embedding] + - should getdata() return isterminal? +""" function transition(state::T, args::NamedTuple )::NamedTuple{(:newNodeKey, :newstate, :progressvalue), Tuple{String, T, Integer}} where {T<:AbstractDict} decisionMakerF::Function = args[:decisionMaker] evaluatorF::Function = args[:evaluator] # reflector::Function = args[:reflector] - context = args[:context] executeSQL::Function = args[:executeSQL] text2textInstructLLM::Function = args[:text2textInstructLLM] # insertSQLVectorDB::Function = args[:insertSQLVectorDB] @@ -955,40 +768,38 @@ function transition(state::T, args::NamedTuple llmFormatName::String = args[:llmFormatName] # getting SQL from vectorDB - thoughtDict = decisionMakerF(state, context, text2textInstructLLM, llmFormatName; - querySQLVectorDBF) - + thoughtDict = decisionMakerF(state, text2textInstructLLM, llmFormatName; + querySQLVectorDBF) + println("\n--- SQLLLM transition() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(thoughtDict) + println("---") + rawresponse = nothing # map action and input() to llm function response = - if thoughtDict["action_name"] == "listalltables" - # deepcopy(state["virtualCustomerChatHistory"]) because I want to keep it clean - # so that other simulation start from this same node is not contaminated with actioninput - listAllTable_json(executeSQL) - elseif thoughtDict["action_name"] == "TABLEINFO" - input = thoughtDict["action_input"] - tableinfo(executeSQL, input) - elseif thoughtDict[:action_name] == "RUNSQL" + if thoughtDict["action_name"] == "RUNSQL" response = SQLexecution(executeSQL, thoughtDict["action_input"]) - if response["success"] - extracted = extractContent_dataframe(response["result"], text2textInstructLLM, - thoughtDict["action_input"], llmFormatName) + if response[:success] + thoughtDict["action_result"] = GeneralUtils.dfToString(response[:result]) + rawresponse = response[:result] (rawresponse=response[:result], result=extracted, errormsg=nothing, success=true) else + thoughtDict["action_result"] = response[:errormsg] + rawresponse = nothing (result=nothing, errormsg=response[:errormsg], success=false) end else - error("undefined LLM function. Requesting $(thoughtDict[:action_name])") + error("undefined LLM function. Requesting $(thoughtDict["action_name"])") end - # this section allow LLM functions above to have different return values. - success::Bool = haskey(response, "success") ? response["success"] : false - result = success ? response["result"] : response["errormsg"] - rawresponse = haskey(response, "rawresponse") ? response["rawresponse"] : nothing - select = haskey(response, "select") ? response["select"] : nothing - reward::Integer = haskey(response, "reward") ? response["reward"] : 0 - isterminal::Bool = haskey(response, "isterminal") ? response["isterminal"] : false - newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON3.write(result), select, reward, isterminal) - progressvalue::Integer = evaluatorF(newstate, thoughtDict, text2textInstructLLM, llmFormatName) + # this section allow LLM functions above to have different return values. + success::Bool = haskey(response, :success) ? response[:success] : false + result = success ? response[:result] : response[:errormsg] + select = haskey(response, :select) ? response[:select] : nothing + reward::Integer = haskey(response, :reward) ? response[:reward] : 0 + isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false + newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON.json(result), + select, reward, isterminal) + progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM, llmFormatName) return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue) end @@ -1075,7 +886,7 @@ julia> println(result) ``` # Signature -""" +""" function query(query::T, executeSQL::Function, text2textInstructLLM::Function; insertSQLVectorDB::Union{Function, Nothing}=nothing, similarSQLVectorDB::Union{Function, Nothing}=nothing, @@ -1095,6 +906,59 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; end end + """ + chathistory= [ + Dict( + "role" => "system", + "content" => [ + Dict("type" => "text", "text" => "You are a helpful assistant"), + ] + ), + ] + """ + + systemmsg = + """ + + - RUNSQL, which you can use to execute SQL against the database. + action_input for this function must be a single SQL query to be executed against the database. + For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. + Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. + + + At each round of conversation, you will be given the following: + - user question + You are working under your mentor supervision and you are also eager to improve your helpfulness. + + + Consult the database search guidelines. Then find the data from a database to satisfy the user's question. + + + Fulfill the objective. + + + - Keep SQL queries focused only on the provided information. + - Do not create any table in the database + - A junction table can be used to link tables together. Another use case is for filtering data. + - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. + - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. + - If there is no search result from the database, remove the restrictive criteria until a search result is available, and proceed from there. + + + 1) plan: Based on the current situation, state a complete action plan to complete the task. Be specific. + 2) action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the available_actions name + 3) action_input: The input to the action you are about to perform according to your plan. + After the action is executed you gets "action_result". It is the output from the action you selected. + + + "plan": "...", + "action_name": "...", + "action_input": "..." + + """ + + + # do MCTS if no data in the database # add extra context for Evaluator so that it knows the observation is from seaching a database initialstate = Dict{String, Any}( @@ -1104,22 +968,35 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; "evaluationscore"=> 0, "suggestion"=> "None", "accepted_as_answer"=> "No", - "lesson"=> nothing, - - # contain question, thought_1, action_1, observation_1, thought_2, ... - "thoughtHistory"=> OrderedDict{String, Any}( - #[] :recap=>, - "question"=> query, - ), + "chathistory"=> Vector{Dict{String, Any}}(), # store system, user and assistant msg + "question"=> query, + "context"=> Dict{String, Any}(), + "action_history"=> OrderedDict{String, Any}( + # "1"=> Dict("plan"=> "...", "action_name"=> "...", "action_input"=> "...", "action_result"=> "..."), + # "2"=> Dict("plan"=> "...", "action_name"=> "...", "action_input"=> "...", "action_result"=> "..."), + # ... + ), ) - # context = Dict( - # :tablelist => listAllTable_str(executeSQL)[:result] - # ) + + systemmsg_dict = Dict( + "role" => "system", + "content" => [ + Dict("type" => "text", "text" => systemmsg), + ] + ) + usermsg = Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => query), + ] + ) + push!(initialstate["chathistory"], systemmsg_dict) + push!(initialstate["chathistory"], usermsg) + #XXX find a way to recreate the schema from a existing database - context = Dict( - :tablelist => - """ - create table customer ( + table_schema = + """ + create table customer ( customer_id uuid primary key default gen_random_uuid (), customer_firstname varchar(128), customer_lastname varchar(128), @@ -1167,6 +1044,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; savoriness integer, bitterness integer, serving_temperature integer, + image_url jsonb, note text, other_attributes jsonb, @@ -1181,7 +1059,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; wine_name varchar(128) not null, winery varchar(128) not null, vintage integer not null, - region varchar(128) not null, -- A field used to store the name of a wine-producing region, such as Napa Valley (California), Bordeaux, Champagne, Tuscany, etc. + region varchar(128) not null, country varchar(128) not null, wine_type varchar(128) not null, grape varchar(128) not null, @@ -1191,7 +1069,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; tannin integer, acidity integer, fizziness integer, - tasting_notes text, -- A field used to record the distinctive flavors of wine such as floral, citrus, apple, earthy, daisy, etc. + tasting_notes text, + image_url jsonb, + manufacturer_sku text, note text, other_attributes jsonb, @@ -1231,22 +1111,22 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; updated_time timestamptz default current_timestamp ); """ - ) + + initialstate["context"]["table_schema"] = table_schema transitionargs = ( - decisionMaker=decisionMaker, - evaluator=evaluator, - reflector=reflector, - context=context, - executeSQL=executeSQL, - text2textInstructLLM=text2textInstructLLM, - querySQLVectorDB=similarSQLVectorDB, - insertSQLVectorDB=insertSQLVectorDB, - llmFormatName=llmFormatName + executeSQL=executeSQL, + decisionMaker=decisionMaker, + evaluator=evaluator, + reflector=reflector, + text2textInstructLLM=text2textInstructLLM, + querySQLVectorDB=similarSQLVectorDB, + insertSQLVectorDB=insertSQLVectorDB, + llmFormatName=llmFormatName ) earlystop(state) = state["reward"] >= 8 ? true : false - + root, _, resultState, highValueState = LLMMCTS.runMCTS(initialstate, transition, transitionargs; horizontalSampleExpansionPhase=1, @@ -1257,33 +1137,44 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; earlystop=earlystop, saveSimulatedNode=true, multithread=false) - + # compare all high value state answer then select the best one if length(highValueState) > 1 - # open("/appfolder/app/highValueState.json", "w") do io - # JSON3.pretty(io, highValueState) - # end selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName) resultState = highValueState[selected] end - latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation") -action_input = "action_input_$latestInd" # latest sql - sql = resultState["thoughtHistory"][action_input] - extractedTableContent = resultState["thoughtHistory"][latestKey] + + + println("\n--- SQLLLM query() resultState ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(resultState) + println("---") + + max_ind = + if length(resultState["action_history"]) == 0 + 0 + else + k = keys(resultState["action_history"]) + maximum(parse.(Int, k)) + end + latest_action = resultState["action_history"]["$max_ind"] + sql = latest_action["action_input"] # add to vectorDB only if the answer is achieved and the state is terminal if insertSQLVectorDB !== nothing && resultState["isterminal"] == true && - resultState["rawresponse"] !== nothing + resultState["accepted_as_answer"] == "yes" - insertSQLVectorDB(resultState["thoughtHistory"]["question"], sql) + insertSQLVectorDB(resultState["question"], sql) end - - if extractedTableContent === nothing + + if latest_action["action_result"] === nothing println("\nSQLLLM query() return nothing ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - - result = (text=extractedTableContent, rawresponse=resultState["rawresponse"]) - + #WORKING 1 + error("SQLLLM query() end") + result = (text=latest_action["action_result"], rawresponse=resultState["rawresponse"]) + println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("---") + error("SQLLLM query() end") return result end @@ -1304,31 +1195,20 @@ julia> function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::T2, select::Union{T3, Nothing}, reward::T3, isterminal::Bool )::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{String, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict} - - keys = [:plan, :action_name, :action_input, :observation] - # latestKeys = [] - currentstate_latestKey, currentstate_latestIndice = - GeneralUtils.findHighestIndexKey(currentstate["thoughtHistory"], keys[1]) - nextindice = currentstate_latestKey !== nothing ? currentstate_latestIndice + 1 : 1 - # currentstate_latestKey == "NA" ? 1 : currentstate_latestIndice + 1 - - currentstate_latestKey = makekey.(keys, nextindice) - - # add Thought, action, observation to thoughtHistory newstate = deepcopy(currentstate) - for (x, y) in zip(keys, currentstate_latestKey) - if x != "observation" - newstate["thoughtHistory"][y] = thoughtDict[x] + max_ind = + if length(newstate["action_history"]) == 0 + 0 else - newstate["thoughtHistory"][y] = response + k = keys(newstate["action_history"]) + maximum(parse.(Int, k)) end - end - -newstate["reward"] = reward - newstate["select"] = select - newstate["isterminal"] = isterminal - newstate["rawresponse"] = rawresponse # whatever return from action + newstate["action_history"]["$(max_ind + 1)"] = thoughtDict + newstate["reward"] = reward + newstate["select"] = select + newstate["isterminal"] = isterminal + newstate["rawresponse"] = rawresponse # whatever return from action newNodeKey = GeneralUtils.uuid4snakecase() @@ -1402,7 +1282,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function, dictkey = ["q1"] workprogress = "" - for (k, v) in state["thoughtHistory"] + for (k, v) in state["action_history"] if k ∉ ["query"] workprogress *= "$k: $v\n" end @@ -1415,7 +1295,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function, usermsg = """ $(context["tablelist"]) - User query: $(state["thoughtHistory"]["question"]) + User query: $(state["action_history"]["question"]) Example: $similarSQL Your work progress: $workprogress P.S. $errornote diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 17fc789..6d16059 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -518,8 +518,8 @@ function SQLexecution(executeSQL::Function, sql::T row, column = tablesize if row == 0 error("\nThe resulting table has 0 row. Please try again.") - elseif column > 30 - error("\nSQL execution failed. An unexpected error occurred. Please try again.") + elseif column > 50 + error("\nSQL execution success but there are more than 50 rows Please be more specific.") end df1 = @@ -821,7 +821,7 @@ julia> # Notes - The function makes up to 10 attempts to get a valid response from the LLM -- Each state in highValueStateList should contain a thoughtHistory with action_input and observation +- Each state in highValueStateList should contain a action_history with action_input and observation - The LLM evaluates attempts based on accuracy and relevance to the original question """ function compareState(question::String, highValueStateList::Vector{T}, @@ -866,14 +866,14 @@ potentialSolution = [] keys = ["action_input", "observation"] # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution for state in highValueStateList - thoughtHistory = state["thoughtHistory"] + action_history = state["action_history"] _, currentstate_latestIndice = - GeneralUtils.findHighestIndexKey(thoughtHistory, keys[1]) + GeneralUtils.findHighestIndexKey(action_history, keys[1]) latestKeys = makekey.(keys, currentstate_latestIndice) d = Dict() # get the last action_name, action_input, observation of currentstate for (i,v) in enumerate(keys) - d[v] = thoughtHistory[latestKeys[i]] + d[v] = action_history[latestKeys[i]] end push!(potentialSolution, d) end diff --git a/test/runtests.jl b/test/runtests.jl index 4ef0327..8498517 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -137,7 +137,7 @@ end function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString} tablename = "sqlllm_decision_repository" # get embedding of the query - # query = state[:thoughtHistory][:question] + # query = state[:action_history][:question] df = findSimilarTextFromVectorDB(query, tablename, "function_input_embedding", executeSQLVectorDB) row, col = size(df) @@ -352,7 +352,7 @@ SELECT * FROM wine WHERE wine_type = 'red' AND country = 'France' AND sweetness # :evaluation => # "The user's question is to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. The thought is correct in identifying the conditions needed to filter the wine table. The action taken is to execute a SQL query to retrieve the desired data, which is also correct. The observation provides a search summary and two search results that match the user's question. Each result includes details about the wine such as ID, name, brand, manufacturer, region, country, type, grape variety, serving temperature, intensity, sweetness, tannin, and acidity.", # :accepted_as_answer => "Yes", -# :thoughtHistory => +# :action_history => # OrderedDict{String, Any}("question" => "Search the database for wine_type: white, country: France, sweetness: 1", "thought_1" => "The user wants to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. To achieve this, we need to filter the wine table based on these conditions.", "action_name_1" => "GETDATA", "action_input_1" => "SELECT * FROM wine WHERE wine.wine_type = 'white' AND wine.country = 'France' AND wine.sweetness = 1;", "observation_1" => "\"Search summary: The resulting table represents wines.\\nSearch result: 1) wine_id: 5b6b6df9-d87c-4f33-8995-7249c2ecc917, wine_name: corton-charlemagne grand cru, brand: domaine des croix, manufacturer: domaine des croix, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n2) wine_id: 1ad27d16-ef64-4907-acf1-40631630c143, wine_name: puligny-montrachet 1er cru 'les demoiselles', brand: amiot guy, manufacturer: amiot guy, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n\\n\""), # :evaluationscore => 9, # :select => nothing,