From bf223b64b2263fcecfb654c27aae2986570441d9 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Sun, 27 Apr 2025 22:32:22 +0700 Subject: [PATCH] update --- src/interface.jl | 80 ++++++++++++++++++++++------------------------ src/llmfunction.jl | 15 +++++---- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 549cdab..d44d09e 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -186,7 +186,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, end response = nothing # store for show when error msg show up - errornote = "" + errornote = "N/A" # provide similar sql only for the first attempt similarSQL_ = "None" @@ -220,7 +220,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, Your work progress: $workprogress Evaluation: $(state[:evaluation]) Suggestion: $(state[:suggestion]) - $errornote + P.S. $errornote """ _prompt = @@ -361,7 +361,7 @@ julia> # Signature """ -function evaluator(state::T1, text2textInstructLLM::Function +function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10 ) where {T1<:AbstractDict} systemmsg = @@ -432,14 +432,11 @@ function evaluator(state::T1, text2textInstructLLM::Function end errornote = "" - - for attempt in 1:10 - errorFlag = false - + for attempt in 1:maxattempt usermsg = """ Trajectory: $thoughthistory - Error_note: $errornote + P.S. $errornote """ _prompt = @@ -464,10 +461,12 @@ function evaluator(state::T1, text2textInstructLLM::Function # check whether response has all header detected_kw = GeneralUtils.detect_keyword(header, response) if 0 ∈ values(detected_kw) - errornote = "\nSQL evaluator() response does not have all header" + errornote = "Your previous attempt does not have all answer points" + println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue elseif sum(values(detected_kw)) > length(header) - errornote = "\nSQL evaluator() response has duplicated header" + errornote = "Your previous attempt has duplicated answer point" + println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end @@ -478,13 +477,17 @@ function evaluator(state::T1, text2textInstructLLM::Function try responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5 catch + errornote = "Your previous attempt's score has wrong format" + println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end accepted_as_answer::AbstractString = responsedict[:accepted_as_answer] if accepted_as_answer ∉ ["Yes", "No"] # [PENDING] add errornote into the prompt - error("generated accepted_as_answer has wrong format") + errornote = "Your previous attempt's accepted_as_answer has wrong format" + println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue end # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension) @@ -1001,13 +1004,13 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; root, _, resultState, highValueState = LLMMCTS.runMCTS(initialstate, transition, transitionargs; horizontalSampleExpansionPhase=3, - horizontalSampleSimulationPhase=2, + horizontalSampleSimulationPhase=3, maxSimulationDepth=5, maxiterations=1, explorationweight=1.0, earlystop=earlystop, saveSimulatedNode=true, - multithread=true) + multithread=false) # compare all high value state answer then select the best one if length(highValueState) > 0 @@ -1088,7 +1091,7 @@ end function generatequestion(state::T1, context, text2textInstructLLM::Function; - similarSQL::Union{T2, Nothing}=nothing + similarSQL::Union{T2, Nothing}=nothing, maxattempt=10 )::String where {T1<:AbstractDict, T2<:AbstractString} similarSQL = @@ -1161,14 +1164,14 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; response = nothing # store for show when error msg show up errornote = "" - for attempt in 1:10 + for attempt in 1:maxattempt usermsg = """ $(context[:tablelist]) User query: $(state[:thoughtHistory][:question]) Example: $similarSQL Your work progress: $workprogress - $errornote + P.S. $errornote """ _prompt = @@ -1180,33 +1183,26 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; # put in model format prompt = GeneralUtils.formatLLMtext(_prompt, "granite3") - try - response = text2textInstructLLM(prompt, modelsize="medium") - response = GeneralUtils.deFormatLLMtext(response, "granite3") - - # check if response is valid - q_number = count("Q", response) - if q_number < 1 - errornote = "too few question" - error("too few questions only $q_number questions are generated") - end - if occursin('`', response) - response = replace(response, '`'=>"") - end - - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) - response = "Q1: " * responsedict[:q1] - # println("\nERROR SQLLLM generatequestion() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - pprintln(Dict(responsedict)) - return response - catch e - io = IOBuffer() - showerror(io, e) - errorMsg = String(take!(io)) - st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) - println("\nERROR SQLLLM generatequestion() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + response = text2textInstructLLM(prompt, modelsize="medium") + response = GeneralUtils.deFormatLLMtext(response, "granite3") + + # check if response is valid + q_number = count("Q", response) + if q_number < 1 + errornote = "Your previous attempt has too few question." + println("\nERROR YiemAgent generatequestion(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue end + if occursin('`', response) + response = replace(response, '`'=>"") + end + + responsedict = GeneralUtils.textToDict(response, header; + dictKey=dictkey, symbolkey=true) + response = "Q1: " * responsedict[:q1] + println("\nSQLLLM generatequestion() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(Dict(responsedict)) + return response end error("generatequestion failed to generate a thought ", response) end diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 1777c88..dac143c 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -856,7 +856,7 @@ function compareState(question::String, highValueStateList::Vector{T}, Result: 50 red wines Comparison: The second attempt counts only German red wines while the first attempt includes all German wines. Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines. - Selected_response_number: 1 + Selected_response_number:1 Let's begin! @@ -908,6 +908,7 @@ function compareState(question::String, highValueStateList::Vector{T}, """ Question: $question Attempts: $potentialSolutionStr + P.S. $errornote """ _prompt = @@ -933,21 +934,23 @@ function compareState(question::String, highValueStateList::Vector{T}, for i in header detected = GeneralUtils.detect_keyword(i, response) if detected === nothing - errornote = "Your previous response didn't provide $i" + errornote = "Your previous attempt didn't provide $i" errorFlag = true end end - if errorFlag + if errorFlag + println("\nERROR SQLLLM compareState() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue # skip to the next iteration end - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) - + responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=true) + responsedict[:selected_response_number] = responsedict[:selected_response_number][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. try responsedict[:selected_response_number] = parse(Int, responsedict[:selected_response_number]) # convert string "5" into integer 5 catch + errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number." + println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end