From 480ecf3e74d0a5ca88b60b1a19bd4862ce751af1 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 14 Jul 2026 18:09:10 +0700 Subject: [PATCH] update --- src/interface.jl | 29 ++++---- src/llmfunction.jl | 178 ++++++++++++++++++++++++--------------------- 2 files changed, 110 insertions(+), 97 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 90f82bd..3ff64a2 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -103,7 +103,7 @@ Dict( # Signature """ function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName::String - ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10 + ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt::Integer=10 )::Dict{String, Any} where {T1<:AbstractDict, T2<:Function} requiredKeys = ["plan", "action_name", "action_input"] @@ -245,14 +245,14 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str - validate whether the SQL query makes sense before accepting it as a valid answer. # you should then respond to the user with - 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. + 1) "trajectory_evaluation", Analyze the trajectory of a solution to answer the user's original question. - Evaluate the correctness of each section and the overall trajectory based on the given question. - Provide detailed reasoning and analysis, focusing on the latest plan, action_name, action_input, and action_result. - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. - Do not generate additional thoughts or actions. - 2) Answer_evaluation: + 2) "answer_evaluation", - Focus only on the matter mentioned in the question and comprehensively analyze how the latest action_input is appropriate. - 3) Accepted_as_answer: Decide whether the latest action_input is technically correct. Can be "yes" or "no" + 3) "accepted_as_answer", Decide whether the latest action_input is technically correct. Can be "yes" or "no" Bad example: question: Find cars with 4 wheels. action_input: INSERT INTO employees @@ -261,7 +261,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str question: Find cars with a sunroof. action_input: SELECT * FROM car_features WHERE has_sunroof = TRUE; - 4) Score: Correctness score s where s is a single integer between 0 to 9. + 4) "score", Correctness score s where s is a single integer between 0 to 9. For example: - 0 indicates that both the trajectory is incorrect, failed or errors and the action_result is incorrect or failed - 4 indicates that the trajectory are correct, but no results are returned. @@ -277,7 +277,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str "accepted_as_answer": "...", "score": "...", "suggestion": "..." - """ requiredKeys = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"] errornote = "" @@ -318,7 +317,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str for attempt in 1:maxattempt response = text2textInstructLLM("random_id", msg) response = GeneralUtils.clean_json_response(response) - response = GeneralUtils.remove_french_accents(response) think, response = GeneralUtils.extractthink(response) response = String(split(response, ", action_result")[1]) # in case LLM generate action_result key which it isn't supposed to @@ -340,7 +338,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str continue end - responsedict["score"] = responsedict["score"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. try responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5 catch @@ -944,7 +941,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; LLMMCTS.runMCTS(initialstate, transition, transitionargs; horizontalSampleExpansionPhase=3, horizontalSampleSimulationPhase=3, - maxSimulationDepth=3, + maxSimulationDepth=5, maxiterations=1, explorationweight=1.0, earlystop=earlystop, @@ -955,10 +952,10 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; # compare all high value state answer then select the best one if length(highValueState) > 1 - selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName) + selected = compareState(query, highValueState, text2textInstructLLM) resultState = highValueState[selected] end - + max_ind = if length(resultState["action_history"]) == 0 0 @@ -967,7 +964,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; maximum(parse.(Int, k)) end latest_action = resultState["action_history"]["$max_ind"] - + # println("\n") + # println(resultState) + # @info "---" @__LINE__ #CHANGE add to vectorDB only if the answer is achieved and the state is terminal sql = latest_action["action_input"] if insertSQLVectorDB !== nothing && resultState["isterminal"] == true && @@ -975,9 +974,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; insertSQLVectorDB(resultState["question"], sql) end - println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # pprintln(resultState) - println("---\n") + # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # # pprintln(resultState) + # println("---\n") return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"]) end diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 2b6110e..042b9a1 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -810,59 +810,71 @@ julia> - The LLM evaluates attempts based on accuracy and relevance to the original question """ function compareState(question::String, highValueStateList::Vector{T}, - text2textInstructLLM::Function, llmFormatName::String + text2textInstructLLM::Function; maxattempt::Integer=10 )::Integer where {T<:AbstractDict} systemmsg = """ - # Your profile: - - You are a helpful assistant - # Situation: - - The user has made multiple attempts to solve the question, resulting in various answers - # Your mission: - - Identify and select the most accurate and relevant response from these multiple results for the user - # At each round of conversation, you will be given the following: - Question: the question the user is trying to answer - Attempt: the user's attempted actions and their corresponding results - # You should then respond to the user with the following: - Comparison: detailed comparison of all results from all attempts from various aspects. - Rationale: a brief explanation of why the selected response is the most accurate and relevant - Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...) - # You should only respond in format as described below: - Comparison: ... - Rationale: ... - Selected_response_number: ... - # Here are some examples: - User's question: "How many German wines do you have?" - Attempt 1) - action_name: RUNSQL - action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' - action_result: 100 wines - Attempt 2) - action_name: RUNSQL - action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red' - action_result: 50 red wines - Comparison: The second attempt counts only German red wines while the first attempt includes all German wines. - Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines. - Selected_response_number:1 + # Your profile: + - You are a helpful assistant - Let's begin! + # Situation: + - The user has made multiple attempts to solve the question, resulting in various answers + + # Your mission: + - Identify and select the most accurate and relevant response from these multiple results for the user + + # At each round of conversation, you will be given the following: + Question: the question the user is trying to answer + Attempt: the user's attempted actions and their corresponding results + + # You should then respond to the user with the following: + 1) "comparison", detailed comparison of all results from all attempts from various aspects. + 2) "rationale", a brief explanation of why the selected response is the most accurate and relevant + 3) "selected_response_number", the number the selected response in the list of results (e.g., 1, 2, 3, ...) + + # you should only respond in JSON format as described below + "comparison": "..." + "rationale": "..." + "selected_response_number": "..." + + # Here are some examples: + Question: "How many German wines do you have?" + Attempt 1) + action_name: RUNSQL + action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' + action_result: 100 wines + Attempt 2) + action_name: RUNSQL + action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red' + action_result: 50 red wines + + "comparison": "The second attempt counts only German red wines while the first attempt includes all German wines." + "rationale": "The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines." + "selected_response_number": "1" """ -potentialSolution = [] + requiredKeys = ["comparison", "rationale", "selected_response_number"] + + potentialSolution = [] includekeys = ["action_name", "action_input", "action_result"] # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution for state in highValueStateList action_history = state["action_history"] - latestKeys = [i for i in keys(d)][end] + latestKeys = [i for i in keys(action_history)][end] d = Dict() # get the last action_name, action_input, observation of currentstate for (i,v) in enumerate(includekeys) - d[v] = action_history[latestKeys[i]] #BUG ERROR: LoadError: KeyError: key "action_input_nothing" not found + latest_action = action_history[latestKeys] + d[v] = latest_action[v] end push!(potentialSolution, d) end + println("\n") + @show potentialSolution + println("--- ", @__FILE__, @__LINE__) + """ # put potential solutions from potentialSolution into the following form Attempt 1) @@ -875,73 +887,75 @@ potentialSolution = [] action_result: ... """ + potentialSolutionStr = "" for (i, state) in enumerate(potentialSolution) potentialSolutionStr *= "Attempt $i)\n" - for k in keys + for k in includekeys potentialSolutionStr *= "$k: $(state[k])\n" - println("") end end - errornote = "N/A" + usermsg = + """ + Question: $question + $potentialSolutionStr + """ - for attempt in 1:10 - errorFlag = false + msg = Dict( + "model" => "gemma-4-E4B-it-UD-Q4_K_XL", + "messages" => [ + Dict( + "role" => "system", + "content" => [ + Dict("type" => "text", "text" => systemmsg), + ] + ), + Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => usermsg), + ] + ), + ], + "temperature" => 0.7 + ) - usermsg = - """ - Question: $question - Attempts: $potentialSolutionStr - P.S. $errornote - """ - - _prompt = - [ - Dict(:name=> "system", :text=> systemmsg), - Dict(:name=> "user", :text=> usermsg) - ] - - # put in model format - prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName) - - header = ["Comparison:", "Rationale:", "Selected_response_number:"] - dictkey = ["comparison", "rationale", "selected_response_number"] - - response = text2textInstructLLM(prompt, modelsize="medium") - - # sometime LLM output something like **Comprehension**: which is not expected - response = replace(response, "**"=>"") - response = replace(response, "***"=>"") - response = GeneralUtils.deFormatLLMtext(response, llmFormatName) + for attempt in 1:maxattempt + response = text2textInstructLLM("random_id", msg) + response = GeneralUtils.clean_json_response(response) + response = GeneralUtils.remove_french_accents(response) think, response = GeneralUtils.extractthink(response) - - # check whether response has all header - detected_kw = GeneralUtils.detectKeywordVariation(header, response) - missingkeys = [k for (k, v) in detected_kw if v === nothing] - if !isempty(missingkeys) - errornote = "$missingkeys are missing from your previous response" - println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum([length(i) for i in values(detected_kw)]) > length(header) - errornote = "\nYour previous attempt has duplicated points according to the required response format" - println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + response = String(split(response, ", action_result")[1]) # in case LLM generate action_result key which it isn't supposed to + response = strip(response) + responsedict = nothing + try + _responsedict = JSON.parse(response) + responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys) + catch + println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=false) + # check whether all answer's key points are in responsedict + ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys) + if !ispass + errornote = errormsg + println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") + continue + end -responsedict["selected_response_number"] = responsedict["selected_response_number"][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. - try - responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5 + try + responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5 catch errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number." println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - pprintln(Dict(responsedict)) + # println("\n~~~ compareState() ") + # pprintln(Dict(responsedict)) + # println("---\n", @__FILE__, ":", @__LINE__) return responsedict["selected_response_number"] end