update
This commit is contained in:
+13
-14
@@ -103,7 +103,7 @@ Dict(
|
||||
# Signature
|
||||
"""
|
||||
function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName::String
|
||||
; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
|
||||
; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt::Integer=10
|
||||
)::Dict{String, Any} where {T1<:AbstractDict, T2<:Function}
|
||||
|
||||
requiredKeys = ["plan", "action_name", "action_input"]
|
||||
@@ -245,14 +245,14 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
- validate whether the SQL query makes sense before accepting it as a valid answer.
|
||||
|
||||
# you should then respond to the user with
|
||||
1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
|
||||
1) "trajectory_evaluation", Analyze the trajectory of a solution to answer the user's original question.
|
||||
- Evaluate the correctness of each section and the overall trajectory based on the given question.
|
||||
- Provide detailed reasoning and analysis, focusing on the latest plan, action_name, action_input, and action_result.
|
||||
- Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached.
|
||||
- Do not generate additional thoughts or actions.
|
||||
2) Answer_evaluation:
|
||||
2) "answer_evaluation",
|
||||
- Focus only on the matter mentioned in the question and comprehensively analyze how the latest action_input is appropriate.
|
||||
3) Accepted_as_answer: Decide whether the latest action_input is technically correct. Can be "yes" or "no"
|
||||
3) "accepted_as_answer", Decide whether the latest action_input is technically correct. Can be "yes" or "no"
|
||||
Bad example:
|
||||
question: Find cars with 4 wheels.
|
||||
action_input: INSERT INTO employees
|
||||
@@ -261,7 +261,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
question: Find cars with a sunroof.
|
||||
action_input: SELECT * FROM car_features
|
||||
WHERE has_sunroof = TRUE;
|
||||
4) Score: Correctness score s where s is a single integer between 0 to 9.
|
||||
4) "score", Correctness score s where s is a single integer between 0 to 9.
|
||||
For example:
|
||||
- 0 indicates that both the trajectory is incorrect, failed or errors and the action_result is incorrect or failed
|
||||
- 4 indicates that the trajectory are correct, but no results are returned.
|
||||
@@ -277,7 +277,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
"accepted_as_answer": "...",
|
||||
"score": "...",
|
||||
"suggestion": "..."
|
||||
|
||||
"""
|
||||
requiredKeys = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
|
||||
errornote = ""
|
||||
@@ -318,7 +317,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
for attempt in 1:maxattempt
|
||||
response = text2textInstructLLM("random_id", msg)
|
||||
response = GeneralUtils.clean_json_response(response)
|
||||
|
||||
response = GeneralUtils.remove_french_accents(response)
|
||||
think, response = GeneralUtils.extractthink(response)
|
||||
response = String(split(response, ", action_result")[1]) # in case LLM generate action_result key which it isn't supposed to
|
||||
@@ -340,7 +338,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
|
||||
continue
|
||||
end
|
||||
|
||||
responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
|
||||
try
|
||||
responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5
|
||||
catch
|
||||
@@ -944,7 +941,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
LLMMCTS.runMCTS(initialstate, transition, transitionargs;
|
||||
horizontalSampleExpansionPhase=3,
|
||||
horizontalSampleSimulationPhase=3,
|
||||
maxSimulationDepth=3,
|
||||
maxSimulationDepth=5,
|
||||
maxiterations=1,
|
||||
explorationweight=1.0,
|
||||
earlystop=earlystop,
|
||||
@@ -955,7 +952,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
|
||||
# compare all high value state answer then select the best one
|
||||
if length(highValueState) > 1
|
||||
selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
|
||||
selected = compareState(query, highValueState, text2textInstructLLM)
|
||||
resultState = highValueState[selected]
|
||||
end
|
||||
|
||||
@@ -967,7 +964,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
maximum(parse.(Int, k))
|
||||
end
|
||||
latest_action = resultState["action_history"]["$max_ind"]
|
||||
|
||||
# println("\n")
|
||||
# println(resultState)
|
||||
# @info "---" @__LINE__
|
||||
#CHANGE add to vectorDB only if the answer is achieved and the state is terminal
|
||||
sql = latest_action["action_input"]
|
||||
if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
|
||||
@@ -975,9 +974,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
insertSQLVectorDB(resultState["question"], sql)
|
||||
end
|
||||
|
||||
println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# pprintln(resultState)
|
||||
println("---\n")
|
||||
# println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# # pprintln(resultState)
|
||||
# println("---\n")
|
||||
|
||||
return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"])
|
||||
end
|
||||
|
||||
+96
-82
@@ -810,59 +810,71 @@ julia>
|
||||
- The LLM evaluates attempts based on accuracy and relevance to the original question
|
||||
"""
|
||||
function compareState(question::String, highValueStateList::Vector{T},
|
||||
text2textInstructLLM::Function, llmFormatName::String
|
||||
text2textInstructLLM::Function; maxattempt::Integer=10
|
||||
)::Integer where {T<:AbstractDict}
|
||||
|
||||
systemmsg =
|
||||
"""
|
||||
# Your profile:
|
||||
- You are a helpful assistant
|
||||
# Situation:
|
||||
- The user has made multiple attempts to solve the question, resulting in various answers
|
||||
# Your mission:
|
||||
- Identify and select the most accurate and relevant response from these multiple results for the user
|
||||
# At each round of conversation, you will be given the following:
|
||||
Question: the question the user is trying to answer
|
||||
Attempt: the user's attempted actions and their corresponding results
|
||||
# You should then respond to the user with the following:
|
||||
Comparison: detailed comparison of all results from all attempts from various aspects.
|
||||
Rationale: a brief explanation of why the selected response is the most accurate and relevant
|
||||
Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
|
||||
# You should only respond in format as described below:
|
||||
Comparison: ...
|
||||
Rationale: ...
|
||||
Selected_response_number: ...
|
||||
# Here are some examples:
|
||||
User's question: "How many German wines do you have?"
|
||||
Attempt 1)
|
||||
action_name: RUNSQL
|
||||
action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
|
||||
action_result: 100 wines
|
||||
Attempt 2)
|
||||
action_name: RUNSQL
|
||||
action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
|
||||
action_result: 50 red wines
|
||||
Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
|
||||
Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
|
||||
Selected_response_number:1
|
||||
# Your profile:
|
||||
- You are a helpful assistant
|
||||
|
||||
Let's begin!
|
||||
# Situation:
|
||||
- The user has made multiple attempts to solve the question, resulting in various answers
|
||||
|
||||
# Your mission:
|
||||
- Identify and select the most accurate and relevant response from these multiple results for the user
|
||||
|
||||
# At each round of conversation, you will be given the following:
|
||||
Question: the question the user is trying to answer
|
||||
Attempt: the user's attempted actions and their corresponding results
|
||||
|
||||
# You should then respond to the user with the following:
|
||||
1) "comparison", detailed comparison of all results from all attempts from various aspects.
|
||||
2) "rationale", a brief explanation of why the selected response is the most accurate and relevant
|
||||
3) "selected_response_number", the number the selected response in the list of results (e.g., 1, 2, 3, ...)
|
||||
|
||||
# you should only respond in JSON format as described below
|
||||
"comparison": "..."
|
||||
"rationale": "..."
|
||||
"selected_response_number": "..."
|
||||
|
||||
# Here are some examples:
|
||||
Question: "How many German wines do you have?"
|
||||
Attempt 1)
|
||||
action_name: RUNSQL
|
||||
action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
|
||||
action_result: 100 wines
|
||||
Attempt 2)
|
||||
action_name: RUNSQL
|
||||
action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
|
||||
action_result: 50 red wines
|
||||
|
||||
"comparison": "The second attempt counts only German red wines while the first attempt includes all German wines."
|
||||
"rationale": "The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines."
|
||||
"selected_response_number": "1"
|
||||
"""
|
||||
|
||||
potentialSolution = []
|
||||
requiredKeys = ["comparison", "rationale", "selected_response_number"]
|
||||
|
||||
potentialSolution = []
|
||||
includekeys = ["action_name", "action_input", "action_result"]
|
||||
# extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
|
||||
for state in highValueStateList
|
||||
action_history = state["action_history"]
|
||||
latestKeys = [i for i in keys(d)][end]
|
||||
latestKeys = [i for i in keys(action_history)][end]
|
||||
d = Dict()
|
||||
# get the last action_name, action_input, observation of currentstate
|
||||
for (i,v) in enumerate(includekeys)
|
||||
d[v] = action_history[latestKeys[i]] #BUG ERROR: LoadError: KeyError: key "action_input_nothing" not found
|
||||
latest_action = action_history[latestKeys]
|
||||
d[v] = latest_action[v]
|
||||
end
|
||||
push!(potentialSolution, d)
|
||||
end
|
||||
|
||||
println("\n")
|
||||
@show potentialSolution
|
||||
println("--- ", @__FILE__, @__LINE__)
|
||||
|
||||
"""
|
||||
# put potential solutions from potentialSolution into the following form
|
||||
Attempt 1)
|
||||
@@ -875,73 +887,75 @@ potentialSolution = []
|
||||
action_result:
|
||||
...
|
||||
"""
|
||||
|
||||
potentialSolutionStr = ""
|
||||
for (i, state) in enumerate(potentialSolution)
|
||||
potentialSolutionStr *= "Attempt $i)\n"
|
||||
for k in keys
|
||||
for k in includekeys
|
||||
potentialSolutionStr *= "$k: $(state[k])\n"
|
||||
println("")
|
||||
end
|
||||
end
|
||||
|
||||
errornote = "N/A"
|
||||
usermsg =
|
||||
"""
|
||||
Question: $question
|
||||
$potentialSolutionStr
|
||||
"""
|
||||
|
||||
for attempt in 1:10
|
||||
errorFlag = false
|
||||
msg = Dict(
|
||||
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||||
"messages" => [
|
||||
Dict(
|
||||
"role" => "system",
|
||||
"content" => [
|
||||
Dict("type" => "text", "text" => systemmsg),
|
||||
]
|
||||
),
|
||||
Dict(
|
||||
"role" => "user",
|
||||
"content" => [
|
||||
Dict("type" => "text", "text" => usermsg),
|
||||
]
|
||||
),
|
||||
],
|
||||
"temperature" => 0.7
|
||||
)
|
||||
|
||||
usermsg =
|
||||
"""
|
||||
Question: $question
|
||||
Attempts: $potentialSolutionStr
|
||||
P.S. $errornote
|
||||
"""
|
||||
|
||||
_prompt =
|
||||
[
|
||||
Dict(:name=> "system", :text=> systemmsg),
|
||||
Dict(:name=> "user", :text=> usermsg)
|
||||
]
|
||||
|
||||
# put in model format
|
||||
prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
|
||||
|
||||
header = ["Comparison:", "Rationale:", "Selected_response_number:"]
|
||||
dictkey = ["comparison", "rationale", "selected_response_number"]
|
||||
|
||||
response = text2textInstructLLM(prompt, modelsize="medium")
|
||||
|
||||
# sometime LLM output something like **Comprehension**: which is not expected
|
||||
response = replace(response, "**"=>"")
|
||||
response = replace(response, "***"=>"")
|
||||
response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
|
||||
for attempt in 1:maxattempt
|
||||
response = text2textInstructLLM("random_id", msg)
|
||||
response = GeneralUtils.clean_json_response(response)
|
||||
response = GeneralUtils.remove_french_accents(response)
|
||||
think, response = GeneralUtils.extractthink(response)
|
||||
|
||||
# check whether response has all header
|
||||
detected_kw = GeneralUtils.detectKeywordVariation(header, response)
|
||||
missingkeys = [k for (k, v) in detected_kw if v === nothing]
|
||||
if !isempty(missingkeys)
|
||||
errornote = "$missingkeys are missing from your previous response"
|
||||
println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
elseif sum([length(i) for i in values(detected_kw)]) > length(header)
|
||||
errornote = "\nYour previous attempt has duplicated points according to the required response format"
|
||||
println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
response = String(split(response, ", action_result")[1]) # in case LLM generate action_result key which it isn't supposed to
|
||||
response = strip(response)
|
||||
responsedict = nothing
|
||||
try
|
||||
_responsedict = JSON.parse(response)
|
||||
responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
|
||||
catch
|
||||
println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
end
|
||||
|
||||
responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=false)
|
||||
# check whether all answer's key points are in responsedict
|
||||
ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
|
||||
if !ispass
|
||||
errornote = errormsg
|
||||
println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
|
||||
continue
|
||||
end
|
||||
|
||||
responsedict["selected_response_number"] = responsedict["selected_response_number"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
|
||||
try
|
||||
responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
|
||||
try
|
||||
responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
|
||||
catch
|
||||
errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number."
|
||||
println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
continue
|
||||
end
|
||||
|
||||
println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
pprintln(Dict(responsedict))
|
||||
# println("\n~~~ compareState() ")
|
||||
# pprintln(Dict(responsedict))
|
||||
# println("---\n", @__FILE__, ":", @__LINE__)
|
||||
|
||||
return responsedict["selected_response_number"]
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user