diff --git a/Manifest.toml b/Manifest.toml
index 1a41a0e..3b7c046 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -2,7 +2,7 @@
julia_version = "1.12.6"
manifest_format = "2.0"
-project_hash = "ec4f3941a75715b7ba32ddf816f49fe57098c82d"
+project_hash = "7f67bfadc362a449809bc58aae732499b3d18e7f"
[[deps.Accessors]]
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
@@ -269,18 +269,18 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
version = "1.11.0"
[[deps.GeneralUtils]]
-deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
-git-tree-sha1 = "b172f75aa622507027cd269af2cc5f335e9821eb"
+deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
+git-tree-sha1 = "8e0ff2ce28b38f779883fddd2eb6fee3551194ac"
repo-rev = "main"
repo-url = "https://git.yiem.cc/ton/GeneralUtils"
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.4.4"
+version = "0.4.8"
[[deps.HTTP]]
deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
-git-tree-sha1 = "e718a35dd7386ccd6bed64a1d84d661972404b99"
+git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "2.5.1"
+version = "2.5.0"
[[deps.HashArrayMappedTries]]
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
diff --git a/Project.toml b/Project.toml
index a8742ca..a5194f4 100644
--- a/Project.toml
+++ b/Project.toml
@@ -24,5 +24,5 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[compat]
Dates = "1.11.0"
-GeneralUtils = "0.4.4"
+GeneralUtils = "0.4.8"
JSON = "1.6.1"
diff --git a/src/interface.jl b/src/interface.jl
index 2891bf5..0803f2f 100644
--- a/src/interface.jl
+++ b/src/interface.jl
@@ -152,9 +152,9 @@ function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName:
responsedict = nothing
try
_responsedict = JSON.parse(response)
- responsedict = GeneralUtils.dictify(_responsedict, keytype=String)
+ responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
catch
- println("\nERROR decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+ println("\nERROR decisionMaker() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
@@ -162,7 +162,7 @@ function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName:
ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
if !ispass
errornote = errormsg
- println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+ println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
continue
end
@@ -324,9 +324,9 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
responsedict = nothing
try
_responsedict = JSON.parse(response)
- responsedict = GeneralUtils.dictify(_responsedict, keytype=String)
+ responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
catch
- println("\nERROR SQLLLM evaluator() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+ println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
@@ -334,7 +334,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
if !ispass
errornote = errormsg
- println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+ println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
continue
end
@@ -379,197 +379,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
end
error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
end
-# function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String;
-# maxattempt=10
-# ) where {T1<:AbstractDict}
-# systemmsg =
-# """
-# You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions.
-
-# Definitions:
-# "question" is the user's question
-# "plan" is agent's plan to complete the task from the current situation
-# "action_name" is the name of the action taken, which can be one of the following functions:
-# - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
-# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
-# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
-# "action_input" is the input to the action
-# "observation" is result of the preceding immediate action
-
-# At each round of conversation, you will be given the following information:
-# trajectory: A history of how you worked on the question chronologically
-# evaluatee_context: The context that evaluatee use to make a decision
-
-# You must follow the following guidelines:
-# - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
-
-# You should then respond to the user with:
-# 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
-# - Evaluate the correctness of each section and the overall trajectory based on the given question.
-# - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation.
-# - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached.
-# - Do not generate additional thoughts or actions.
-# 2) Answer_evaluation:
-# - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
-# 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no"
-# Bad example (The observation didn't answers the question):
-# question: Find cars with 4 wheels.
-# observation: There are an apple in the table.
-# Good example (The observation answers the question):
-# question: Find cars with a stereo.
-# observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black.
-# 4) Score: Correctness score s where s is a single integer between 0 to 9.
-# For example:
-# - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed
-# - 4 indicates that the trajectory are correct, but no results are returned.
-# - 5 indicates that the trajectory are correct but the observation is incorrect or failed
-# - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question
-# - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
-# - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
-# 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
-
-# You should only respond in format as described below:
-# Trajectory_evaluation: ...
-# Answer_evaluation: ...
-# Accepted_as_answer: ...
-# Score: ...
-# Suggestion: ...
-
-# Let's begin!
-# """
-# #[WORKING] add what I should think --> this will be the think for decisionMaker()
-# header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"]
-# dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
-
-# action_history = ""
-# for (k, v) in state["action_history"]
-# action_history *= "$k: $v\n"
-# end
-
-# errornote = "N/A"
-# for attempt in 1:maxattempt
-# usermsg =
-# """
-#
-# $action_history
-#
-# """
-# context =
-# """
-#
-#
-# thoughtDict[:context]
-#
-# P.S. $errornote
-#
-# """
-
-# unformatPrompt =
-# [
-# Dict(:name => "system", :text => systemmsg),
-# Dict(:name => "user", :text => usermsg)
-# ]
-# # put in model format
-# prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName)
-# # add info
-# prompt = prompt * context
-
-# response = text2textInstructLLM(prompt, modelsize="medium")
-# response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
-# think, response = GeneralUtils.extractthink(response)
-
-# # sometime LLM output something like **Comprehension**: which is not expected
-# response = replace(response, "**"=>"")
-# response = replace(response, "***"=>"")
-
-# # check whether response has all header
-# detected_kw = GeneralUtils.detectKeywordVariation(header, response)
-# missingkeys = [k for (k, v) in detected_kw if v === nothing]
-# if !isempty(missingkeys)
-# errornote = "$missingkeys are missing from your previous response"
-# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-# continue
-# elseif sum([length(i) for i in values(detected_kw)]) > length(header)
-# errornote = "\nYour previous attempt has duplicated points according to the required response format"
-# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-# continue
-# end
-
-# responsedict = GeneralUtils.textToDict(response, header;
-# dictKey=dictkey, symbolkey=false)
-
-# responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
-# try
-# responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5
-# catch
-# errornote = "Your previous attempt's score has wrong format"
-# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-# continue
-# end
-
-# accepted_as_answer::AbstractString = responsedict["accepted_as_answer"]
-
-# if accepted_as_answer ∉ ["yes", "no"]
-# errornote = "Your previous attempt's accepted_as_answer has wrong format"
-# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["accepted_as_answer"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-# continue
-# end
-
-# # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension)
-# state["evaluation"] = "$(responsedict["trajectory_evaluation"]) $(responsedict["answer_evaluation"])"
-# state["evaluationscore"] = responsedict["score"]
-# state["accepted_as_answer"] = responsedict["accepted_as_answer"]
-# state["suggestion"] = responsedict["suggestion"]
-
-# # mark as terminal state when the answer is achieved
-# if accepted_as_answer ∈ ["Yes", "yes"]
-
-# # mark the state as terminal state because the evaluation say so.
-# state["isterminal"] = true
-
-# # evaluation score as reward because different answers hold different value for the user.
-# state["reward"] = responsedict["score"]
-# end
-
-# println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-# pprintln(Dict(responsedict))
-
-# # # store for later training
-# # responsedict[:action_history] = state[:action_history]
-# # responsedict[:system] = systemmsg
-# # responsedict[:usermsg] = usermsg
-# # responsedict[:prompt] = prompt
-# # responsedict[:context] = context
-# # responsedict[:think] = think
-
-# # # read sessionId
-# # sessionid = JSON.parse("/appfolder/app/sessionid.json")
-# # # save to filename ./log/decisionlog.txt
-# # println("saving SQLLLM evaluator() to disk")
-# # filename = "agent_evaluator_log_$(sessionid[:id]).json"
-# # filepath = "/appfolder/app/log/$filename"
-# # # check whether there is a file path exists before writing to it
-# # if !isfile(filepath)
-# # decisionlist = [responsedict]
-# # println("Creating file $filepath")
-# # open(filepath, "w") do io
-# # JSON3.pretty(io, decisionlist)
-# # end
-# # else
-# # # read the file and append new data
-# # decisionlist = copy(JSON.parse(filepath))
-# # push!(decisionlist, responsedict)
-# # println("Appending new data to file $filepath")
-# # open(filepath, "w") do io
-# # JSON3.pretty(io, decisionlist)
-# # end
-# # end
-
-# return responsedict["score"]
-# end
-# error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
-# end
"""
@@ -777,6 +587,10 @@ function transition(state::T, args::NamedTuple
response = nothing
if thoughtDict["action_name"] == "RUNSQL"
response = SQLexecution(executeSQL, thoughtDict["action_input"])
+ # println("\n--- SQLLLM transition() response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+ # println(response)
+ # println("---")
+
else
error("undefined LLM function. Requesting $(thoughtDict["action_name"])")
end
@@ -788,7 +602,11 @@ function transition(state::T, args::NamedTuple
else
evaluatorF(newstate, text2textInstructLLM, llmFormatName)
end
-
+
+ println("\n--- SQLLLM transition() thoughtDict ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+ pprintln(thoughtDict)
+ println("---")
+ # error("SQLLLM transition() end")
return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue)
end
@@ -1102,6 +920,10 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
);
"""
+ # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+ # println("---")
+ # error("SQLLLM query() end")
+
initialstate["context"]["table_schema"] = table_schema
transitionargs = (
@@ -1128,6 +950,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
saveSimulatedNode=true,
multithread=false)
+ # error("SQLLLM query() end")
+
# compare all high value state answer then select the best one
if length(highValueState) > 1
selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
@@ -1144,17 +968,16 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
latest_action = resultState["action_history"]["$max_ind"]
#CHANGE add to vectorDB only if the answer is achieved and the state is terminal
- # sql = latest_action["action_input"]
- # if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
- # resultState["accepted_as_answer"] == "yes"
- # insertSQLVectorDB(resultState["question"], sql)
- # end
-
- # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
- # println(latest_action)
- # println("---")
- # error("SQLLLM query() end")
+ sql = latest_action["action_input"]
+ if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
+ resultState["accepted_as_answer"] == "yes"
+ insertSQLVectorDB(resultState["question"], sql)
+ end
+ println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+ println(resultState["result_raw"])
+ println("---\n")
+
return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"])
end
diff --git a/src/llmfunction.jl b/src/llmfunction.jl
index 33fa2fb..0cf8f60 100644
--- a/src/llmfunction.jl
+++ b/src/llmfunction.jl
@@ -514,10 +514,10 @@ function SQLexecution(executeSQL::Function, sql::T
df
end
result = GeneralUtils.dfToString(df1)
- println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
- println(sql)
- println(df1)
- println("\n")
+ # println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
+ # println(sql)
+ # println(df1)
+ # println("\n")
return (result_str=result, result_raw=df1, success=true, errormsg=nothing)
end
catch e