update
This commit is contained in:
+6
-6
@@ -2,7 +2,7 @@
|
||||
|
||||
julia_version = "1.12.6"
|
||||
manifest_format = "2.0"
|
||||
project_hash = "ec4f3941a75715b7ba32ddf816f49fe57098c82d"
|
||||
project_hash = "7f67bfadc362a449809bc58aae732499b3d18e7f"
|
||||
|
||||
[[deps.Accessors]]
|
||||
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
|
||||
@@ -269,18 +269,18 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
|
||||
version = "1.11.0"
|
||||
|
||||
[[deps.GeneralUtils]]
|
||||
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
|
||||
git-tree-sha1 = "b172f75aa622507027cd269af2cc5f335e9821eb"
|
||||
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
|
||||
git-tree-sha1 = "8e0ff2ce28b38f779883fddd2eb6fee3551194ac"
|
||||
repo-rev = "main"
|
||||
repo-url = "https://git.yiem.cc/ton/GeneralUtils"
|
||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||
version = "0.4.4"
|
||||
version = "0.4.8"
|
||||
|
||||
[[deps.HTTP]]
|
||||
deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
|
||||
git-tree-sha1 = "e718a35dd7386ccd6bed64a1d84d661972404b99"
|
||||
git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55"
|
||||
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||
version = "2.5.1"
|
||||
version = "2.5.0"
|
||||
|
||||
[[deps.HashArrayMappedTries]]
|
||||
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
|
||||
|
||||
+1
-1
@@ -24,5 +24,5 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||
|
||||
[compat]
|
||||
Dates = "1.11.0"
|
||||
GeneralUtils = "0.4.4"
|
||||
GeneralUtils = "0.4.8"
|
||||
JSON = "1.6.1"
|
||||
|
||||
+24
-201
@@ -152,7 +152,7 @@ function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName:
|
||||
responsedict = nothing
|
||||
try
|
||||
_responsedict = JSON.parse(response)
|
||||
responsedict = GeneralUtils.dictify(_responsedict, keytype=String)
|
||||
responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
|
||||
catch
|
||||
println("\nERROR decisionMaker() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
@@ -324,7 +324,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
responsedict = nothing
|
||||
try
|
||||
_responsedict = JSON.parse(response)
|
||||
responsedict = GeneralUtils.dictify(_responsedict, keytype=String)
|
||||
responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
|
||||
catch
|
||||
println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
@@ -379,197 +379,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
end
|
||||
error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
|
||||
end
|
||||
# function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String;
|
||||
# maxattempt=10
|
||||
# ) where {T1<:AbstractDict}
|
||||
|
||||
# systemmsg =
|
||||
# """
|
||||
# You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions.
|
||||
|
||||
# Definitions:
|
||||
# "question" is the user's question
|
||||
# "plan" is agent's plan to complete the task from the current situation
|
||||
# "action_name" is the name of the action taken, which can be one of the following functions:
|
||||
# - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
|
||||
# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
|
||||
# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
|
||||
# "action_input" is the input to the action
|
||||
# "observation" is result of the preceding immediate action
|
||||
|
||||
# At each round of conversation, you will be given the following information:
|
||||
# trajectory: A history of how you worked on the question chronologically
|
||||
# evaluatee_context: The context that evaluatee use to make a decision
|
||||
|
||||
# You must follow the following guidelines:
|
||||
# - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
|
||||
|
||||
# You should then respond to the user with:
|
||||
# 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
|
||||
# - Evaluate the correctness of each section and the overall trajectory based on the given question.
|
||||
# - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation.
|
||||
# - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached.
|
||||
# - Do not generate additional thoughts or actions.
|
||||
# 2) Answer_evaluation:
|
||||
# - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
|
||||
# 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no"
|
||||
# Bad example (The observation didn't answers the question):
|
||||
# question: Find cars with 4 wheels.
|
||||
# observation: There are an apple in the table.
|
||||
# Good example (The observation answers the question):
|
||||
# question: Find cars with a stereo.
|
||||
# observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black.
|
||||
# 4) Score: Correctness score s where s is a single integer between 0 to 9.
|
||||
# For example:
|
||||
# - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed
|
||||
# - 4 indicates that the trajectory are correct, but no results are returned.
|
||||
# - 5 indicates that the trajectory are correct but the observation is incorrect or failed
|
||||
# - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question
|
||||
# - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
|
||||
# - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
|
||||
# 5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
|
||||
|
||||
# You should only respond in format as described below:
|
||||
# Trajectory_evaluation: ...
|
||||
# Answer_evaluation: ...
|
||||
# Accepted_as_answer: ...
|
||||
# Score: ...
|
||||
# Suggestion: ...
|
||||
|
||||
# Let's begin!
|
||||
# """
|
||||
# #[WORKING] add what I should think --> this will be the think for decisionMaker()
|
||||
# header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"]
|
||||
# dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
|
||||
|
||||
# action_history = ""
|
||||
# for (k, v) in state["action_history"]
|
||||
# action_history *= "$k: $v\n"
|
||||
# end
|
||||
|
||||
# errornote = "N/A"
|
||||
# for attempt in 1:maxattempt
|
||||
# usermsg =
|
||||
# """
|
||||
# <trajectory>
|
||||
# $action_history
|
||||
# </trajectory>
|
||||
# """
|
||||
# context =
|
||||
# """
|
||||
# <context>
|
||||
# <evaluatee_context>
|
||||
# thoughtDict[:context]
|
||||
# </evaluatee_context>
|
||||
# P.S. $errornote
|
||||
# </context>
|
||||
# """
|
||||
|
||||
# unformatPrompt =
|
||||
# [
|
||||
# Dict(:name => "system", :text => systemmsg),
|
||||
# Dict(:name => "user", :text => usermsg)
|
||||
# ]
|
||||
# # put in model format
|
||||
# prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName)
|
||||
# # add info
|
||||
# prompt = prompt * context
|
||||
|
||||
# response = text2textInstructLLM(prompt, modelsize="medium")
|
||||
# response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
|
||||
# think, response = GeneralUtils.extractthink(response)
|
||||
|
||||
# # sometime LLM output something like **Comprehension**: which is not expected
|
||||
# response = replace(response, "**"=>"")
|
||||
# response = replace(response, "***"=>"")
|
||||
|
||||
# # check whether response has all header
|
||||
# detected_kw = GeneralUtils.detectKeywordVariation(header, response)
|
||||
# missingkeys = [k for (k, v) in detected_kw if v === nothing]
|
||||
# if !isempty(missingkeys)
|
||||
# errornote = "$missingkeys are missing from your previous response"
|
||||
# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# continue
|
||||
# elseif sum([length(i) for i in values(detected_kw)]) > length(header)
|
||||
# errornote = "\nYour previous attempt has duplicated points according to the required response format"
|
||||
# println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# continue
|
||||
# end
|
||||
|
||||
# responsedict = GeneralUtils.textToDict(response, header;
|
||||
# dictKey=dictkey, symbolkey=false)
|
||||
|
||||
# responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
|
||||
# try
|
||||
# responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5
|
||||
# catch
|
||||
# errornote = "Your previous attempt's score has wrong format"
|
||||
# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# continue
|
||||
# end
|
||||
|
||||
# accepted_as_answer::AbstractString = responsedict["accepted_as_answer"]
|
||||
|
||||
# if accepted_as_answer ∉ ["yes", "no"]
|
||||
# errornote = "Your previous attempt's accepted_as_answer has wrong format"
|
||||
# println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["accepted_as_answer"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# continue
|
||||
# end
|
||||
|
||||
# # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension)
|
||||
# state["evaluation"] = "$(responsedict["trajectory_evaluation"]) $(responsedict["answer_evaluation"])"
|
||||
# state["evaluationscore"] = responsedict["score"]
|
||||
# state["accepted_as_answer"] = responsedict["accepted_as_answer"]
|
||||
# state["suggestion"] = responsedict["suggestion"]
|
||||
|
||||
# # mark as terminal state when the answer is achieved
|
||||
# if accepted_as_answer ∈ ["Yes", "yes"]
|
||||
|
||||
# # mark the state as terminal state because the evaluation say so.
|
||||
# state["isterminal"] = true
|
||||
|
||||
# # evaluation score as reward because different answers hold different value for the user.
|
||||
# state["reward"] = responsedict["score"]
|
||||
# end
|
||||
|
||||
# println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# pprintln(Dict(responsedict))
|
||||
|
||||
# # # store for later training
|
||||
# # responsedict[:action_history] = state[:action_history]
|
||||
# # responsedict[:system] = systemmsg
|
||||
# # responsedict[:usermsg] = usermsg
|
||||
# # responsedict[:prompt] = prompt
|
||||
# # responsedict[:context] = context
|
||||
# # responsedict[:think] = think
|
||||
|
||||
# # # read sessionId
|
||||
# # sessionid = JSON.parse("/appfolder/app/sessionid.json")
|
||||
# # # save to filename ./log/decisionlog.txt
|
||||
# # println("saving SQLLLM evaluator() to disk")
|
||||
# # filename = "agent_evaluator_log_$(sessionid[:id]).json"
|
||||
# # filepath = "/appfolder/app/log/$filename"
|
||||
# # # check whether there is a file path exists before writing to it
|
||||
# # if !isfile(filepath)
|
||||
# # decisionlist = [responsedict]
|
||||
# # println("Creating file $filepath")
|
||||
# # open(filepath, "w") do io
|
||||
# # JSON3.pretty(io, decisionlist)
|
||||
# # end
|
||||
# # else
|
||||
# # # read the file and append new data
|
||||
# # decisionlist = copy(JSON.parse(filepath))
|
||||
# # push!(decisionlist, responsedict)
|
||||
# # println("Appending new data to file $filepath")
|
||||
# # open(filepath, "w") do io
|
||||
# # JSON3.pretty(io, decisionlist)
|
||||
# # end
|
||||
# # end
|
||||
|
||||
# return responsedict["score"]
|
||||
# end
|
||||
# error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
|
||||
# end
|
||||
|
||||
"""
|
||||
|
||||
@@ -777,6 +587,10 @@ function transition(state::T, args::NamedTuple
|
||||
response = nothing
|
||||
if thoughtDict["action_name"] == "RUNSQL"
|
||||
response = SQLexecution(executeSQL, thoughtDict["action_input"])
|
||||
# println("\n--- SQLLLM transition() response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# println(response)
|
||||
# println("---")
|
||||
|
||||
else
|
||||
error("undefined LLM function. Requesting $(thoughtDict["action_name"])")
|
||||
end
|
||||
@@ -789,6 +603,10 @@ function transition(state::T, args::NamedTuple
|
||||
evaluatorF(newstate, text2textInstructLLM, llmFormatName)
|
||||
end
|
||||
|
||||
println("\n--- SQLLLM transition() thoughtDict ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
pprintln(thoughtDict)
|
||||
println("---")
|
||||
# error("SQLLLM transition() end")
|
||||
return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue)
|
||||
end
|
||||
|
||||
@@ -1102,6 +920,10 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
);
|
||||
"""
|
||||
|
||||
# println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# println("---")
|
||||
# error("SQLLLM query() end")
|
||||
|
||||
initialstate["context"]["table_schema"] = table_schema
|
||||
|
||||
transitionargs = (
|
||||
@@ -1128,6 +950,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
saveSimulatedNode=true,
|
||||
multithread=false)
|
||||
|
||||
# error("SQLLLM query() end")
|
||||
|
||||
# compare all high value state answer then select the best one
|
||||
if length(highValueState) > 1
|
||||
selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
|
||||
@@ -1144,16 +968,15 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
latest_action = resultState["action_history"]["$max_ind"]
|
||||
|
||||
#CHANGE add to vectorDB only if the answer is achieved and the state is terminal
|
||||
# sql = latest_action["action_input"]
|
||||
# if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
|
||||
# resultState["accepted_as_answer"] == "yes"
|
||||
# insertSQLVectorDB(resultState["question"], sql)
|
||||
# end
|
||||
sql = latest_action["action_input"]
|
||||
if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
|
||||
resultState["accepted_as_answer"] == "yes"
|
||||
insertSQLVectorDB(resultState["question"], sql)
|
||||
end
|
||||
|
||||
# println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# println(latest_action)
|
||||
# println("---")
|
||||
# error("SQLLLM query() end")
|
||||
println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
println(resultState["result_raw"])
|
||||
println("---\n")
|
||||
|
||||
return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"])
|
||||
end
|
||||
|
||||
+4
-4
@@ -514,10 +514,10 @@ function SQLexecution(executeSQL::Function, sql::T
|
||||
df
|
||||
end
|
||||
result = GeneralUtils.dfToString(df1)
|
||||
println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
|
||||
println(sql)
|
||||
println(df1)
|
||||
println("\n")
|
||||
# println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
|
||||
# println(sql)
|
||||
# println(df1)
|
||||
# println("\n")
|
||||
return (result_str=result, result_raw=df1, success=true, errormsg=nothing)
|
||||
end
|
||||
catch e
|
||||
|
||||
Reference in New Issue
Block a user