diff --git a/Manifest.toml b/Manifest.toml index 1a41a0e..3b7c046 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "ec4f3941a75715b7ba32ddf816f49fe57098c82d" +project_hash = "7f67bfadc362a449809bc58aae732499b3d18e7f" [[deps.Accessors]] deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] @@ -269,18 +269,18 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" [[deps.GeneralUtils]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"] -git-tree-sha1 = "b172f75aa622507027cd269af2cc5f335e9821eb" +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"] +git-tree-sha1 = "8e0ff2ce28b38f779883fddd2eb6fee3551194ac" repo-rev = "main" repo-url = "https://git.yiem.cc/ton/GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -version = "0.4.4" +version = "0.4.8" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"] -git-tree-sha1 = "e718a35dd7386ccd6bed64a1d84d661972404b99" +git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "2.5.1" +version = "2.5.0" [[deps.HashArrayMappedTries]] git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" diff --git a/Project.toml b/Project.toml index a8742ca..a5194f4 100644 --- a/Project.toml +++ b/Project.toml @@ -24,5 +24,5 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] Dates = "1.11.0" -GeneralUtils = "0.4.4" +GeneralUtils = "0.4.8" JSON = "1.6.1" diff --git a/src/interface.jl b/src/interface.jl index 2891bf5..0803f2f 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -152,9 +152,9 @@ function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName: responsedict = nothing try _responsedict = JSON.parse(response) - responsedict = GeneralUtils.dictify(_responsedict, keytype=String) + responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys) catch - println("\nERROR decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR decisionMaker() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end @@ -162,7 +162,7 @@ function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName: ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys) if !ispass errornote = errormsg - println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") + println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") continue end @@ -324,9 +324,9 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str responsedict = nothing try _responsedict = JSON.parse(response) - responsedict = GeneralUtils.dictify(_responsedict, keytype=String) + responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys) catch - println("\nERROR SQLLLM evaluator() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end @@ -334,7 +334,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys) if !ispass errornote = errormsg - println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") + println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") continue end @@ -379,197 +379,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str end error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") end -# function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String; -# maxattempt=10 -# ) where {T1<:AbstractDict} -# systemmsg = -# """ -# You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions. - -# Definitions: -# "question" is the user's question -# "plan" is agent's plan to complete the task from the current situation -# "action_name" is the name of the action taken, which can be one of the following functions: -# - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. -# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. -# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. -# "action_input" is the input to the action -# "observation" is result of the preceding immediate action - -# At each round of conversation, you will be given the following information: -# trajectory: A history of how you worked on the question chronologically -# evaluatee_context: The context that evaluatee use to make a decision - -# You must follow the following guidelines: -# - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. - -# You should then respond to the user with: -# 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. -# - Evaluate the correctness of each section and the overall trajectory based on the given question. -# - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation. -# - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. -# - Do not generate additional thoughts or actions. -# 2) Answer_evaluation: -# - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question -# 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no" -# Bad example (The observation didn't answers the question): -# question: Find cars with 4 wheels. -# observation: There are an apple in the table. -# Good example (The observation answers the question): -# question: Find cars with a stereo. -# observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black. -# 4) Score: Correctness score s where s is a single integer between 0 to 9. -# For example: -# - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed -# - 4 indicates that the trajectory are correct, but no results are returned. -# - 5 indicates that the trajectory are correct but the observation is incorrect or failed -# - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question -# - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. -# - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. -# 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made? - -# You should only respond in format as described below: -# Trajectory_evaluation: ... -# Answer_evaluation: ... -# Accepted_as_answer: ... -# Score: ... -# Suggestion: ... - -# Let's begin! -# """ -# #[WORKING] add what I should think --> this will be the think for decisionMaker() -# header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"] -# dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"] - -# action_history = "" -# for (k, v) in state["action_history"] -# action_history *= "$k: $v\n" -# end - -# errornote = "N/A" -# for attempt in 1:maxattempt -# usermsg = -# """ -# -# $action_history -# -# """ -# context = -# """ -# -# -# thoughtDict[:context] -# -# P.S. $errornote -# -# """ - -# unformatPrompt = -# [ -# Dict(:name => "system", :text => systemmsg), -# Dict(:name => "user", :text => usermsg) -# ] -# # put in model format -# prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName) -# # add info -# prompt = prompt * context - -# response = text2textInstructLLM(prompt, modelsize="medium") -# response = GeneralUtils.deFormatLLMtext(response, llmFormatName) -# think, response = GeneralUtils.extractthink(response) - -# # sometime LLM output something like **Comprehension**: which is not expected -# response = replace(response, "**"=>"") -# response = replace(response, "***"=>"") - -# # check whether response has all header -# detected_kw = GeneralUtils.detectKeywordVariation(header, response) -# missingkeys = [k for (k, v) in detected_kw if v === nothing] -# if !isempty(missingkeys) -# errornote = "$missingkeys are missing from your previous response" -# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# elseif sum([length(i) for i in values(detected_kw)]) > length(header) -# errornote = "\nYour previous attempt has duplicated points according to the required response format" -# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# responsedict = GeneralUtils.textToDict(response, header; -# dictKey=dictkey, symbolkey=false) - -# responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. -# try -# responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5 -# catch -# errornote = "Your previous attempt's score has wrong format" -# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# accepted_as_answer::AbstractString = responsedict["accepted_as_answer"] - -# if accepted_as_answer ∉ ["yes", "no"] -# errornote = "Your previous attempt's accepted_as_answer has wrong format" -# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["accepted_as_answer"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension) -# state["evaluation"] = "$(responsedict["trajectory_evaluation"]) $(responsedict["answer_evaluation"])" -# state["evaluationscore"] = responsedict["score"] -# state["accepted_as_answer"] = responsedict["accepted_as_answer"] -# state["suggestion"] = responsedict["suggestion"] - -# # mark as terminal state when the answer is achieved -# if accepted_as_answer ∈ ["Yes", "yes"] - -# # mark the state as terminal state because the evaluation say so. -# state["isterminal"] = true - -# # evaluation score as reward because different answers hold different value for the user. -# state["reward"] = responsedict["score"] -# end - -# println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# pprintln(Dict(responsedict)) - -# # # store for later training -# # responsedict[:action_history] = state[:action_history] -# # responsedict[:system] = systemmsg -# # responsedict[:usermsg] = usermsg -# # responsedict[:prompt] = prompt -# # responsedict[:context] = context -# # responsedict[:think] = think - -# # # read sessionId -# # sessionid = JSON.parse("/appfolder/app/sessionid.json") -# # # save to filename ./log/decisionlog.txt -# # println("saving SQLLLM evaluator() to disk") -# # filename = "agent_evaluator_log_$(sessionid[:id]).json" -# # filepath = "/appfolder/app/log/$filename" -# # # check whether there is a file path exists before writing to it -# # if !isfile(filepath) -# # decisionlist = [responsedict] -# # println("Creating file $filepath") -# # open(filepath, "w") do io -# # JSON3.pretty(io, decisionlist) -# # end -# # else -# # # read the file and append new data -# # decisionlist = copy(JSON.parse(filepath)) -# # push!(decisionlist, responsedict) -# # println("Appending new data to file $filepath") -# # open(filepath, "w") do io -# # JSON3.pretty(io, decisionlist) -# # end -# # end - -# return responsedict["score"] -# end -# error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") -# end """ @@ -777,6 +587,10 @@ function transition(state::T, args::NamedTuple response = nothing if thoughtDict["action_name"] == "RUNSQL" response = SQLexecution(executeSQL, thoughtDict["action_input"]) + # println("\n--- SQLLLM transition() response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # println(response) + # println("---") + else error("undefined LLM function. Requesting $(thoughtDict["action_name"])") end @@ -788,7 +602,11 @@ function transition(state::T, args::NamedTuple else evaluatorF(newstate, text2textInstructLLM, llmFormatName) end - + + println("\n--- SQLLLM transition() thoughtDict ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(thoughtDict) + println("---") + # error("SQLLLM transition() end") return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue) end @@ -1102,6 +920,10 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; ); """ + # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # println("---") + # error("SQLLLM query() end") + initialstate["context"]["table_schema"] = table_schema transitionargs = ( @@ -1128,6 +950,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; saveSimulatedNode=true, multithread=false) + # error("SQLLLM query() end") + # compare all high value state answer then select the best one if length(highValueState) > 1 selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName) @@ -1144,17 +968,16 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; latest_action = resultState["action_history"]["$max_ind"] #CHANGE add to vectorDB only if the answer is achieved and the state is terminal - # sql = latest_action["action_input"] - # if insertSQLVectorDB !== nothing && resultState["isterminal"] == true && - # resultState["accepted_as_answer"] == "yes" - # insertSQLVectorDB(resultState["question"], sql) - # end - - # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # println(latest_action) - # println("---") - # error("SQLLLM query() end") + sql = latest_action["action_input"] + if insertSQLVectorDB !== nothing && resultState["isterminal"] == true && + resultState["accepted_as_answer"] == "yes" + insertSQLVectorDB(resultState["question"], sql) + end + println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println(resultState["result_raw"]) + println("---\n") + return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"]) end diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 33fa2fb..0cf8f60 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -514,10 +514,10 @@ function SQLexecution(executeSQL::Function, sql::T df end result = GeneralUtils.dfToString(df1) - println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__) - println(sql) - println(df1) - println("\n") + # println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__) + # println(sql) + # println(df1) + # println("\n") return (result_str=result, result_raw=df1, success=true, errormsg=nothing) end catch e