src/interface.jl

@@ -103,7 +103,7 @@ Dict(
 # Signature
 """
 function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName::String
-  ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
+  ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt::Integer=10
   )::Dict{String, Any} where {T1<:AbstractDict, T2<:Function}
 
   requiredKeys = ["plan", "action_name", "action_input"]
@@ -245,14 +245,14 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
     - validate whether the SQL query makes sense before accepting it as a valid answer.
 
   # you should then respond to the user with
-    1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
+    1) "trajectory_evaluation", Analyze the trajectory of a solution to answer the user's original question.
       - Evaluate the correctness of each section and the overall trajectory based on the given question.
       - Provide detailed reasoning and analysis, focusing on the latest plan, action_name, action_input, and action_result.
       - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached.
       - Do not generate additional thoughts or actions.
-    2) Answer_evaluation: 
+    2) "answer_evaluation", 
       - Focus only on the matter mentioned in the question and comprehensively analyze how the latest action_input is appropriate.
-    3) Accepted_as_answer: Decide whether the latest action_input is technically correct. Can be "yes" or "no"
+    3) "accepted_as_answer", Decide whether the latest action_input is technically correct. Can be "yes" or "no"
       Bad example:
         question: Find cars with 4 wheels.
         action_input: INSERT INTO employees 
@@ -261,7 +261,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
         question: Find cars with a sunroof.
         action_input: SELECT * FROM car_features 
                       WHERE has_sunroof = TRUE;
-    4) Score: Correctness score s where s is a single integer between 0 to 9. 
+    4) "score", Correctness score s where s is a single integer between 0 to 9. 
       For example:
       - 0 indicates that both the trajectory is incorrect, failed or errors and the action_result is incorrect or failed
       - 4 indicates that the trajectory are correct, but no results are returned.
@@ -277,7 +277,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
     "accepted_as_answer": "...",
     "score": "...",
     "suggestion": "..."
-
   """ 
   requiredKeys = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
   errornote = ""
@@ -318,7 +317,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
   for attempt in 1:maxattempt
     response = text2textInstructLLM("random_id", msg)
     response = GeneralUtils.clean_json_response(response)
-
     response = GeneralUtils.remove_french_accents(response)
     think, response = GeneralUtils.extractthink(response)
     response = String(split(response, ", action_result")[1])  # in case LLM generate action_result key which it isn't supposed to
@@ -340,7 +338,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
       continue
     end
     
-    responsedict["score"] = responsedict["score"][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
     try
       responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5
     catch
@@ -944,7 +941,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
     LLMMCTS.runMCTS(initialstate, transition, transitionargs;
                   horizontalSampleExpansionPhase=3, 
                   horizontalSampleSimulationPhase=3,
-                  maxSimulationDepth=3,
+                  maxSimulationDepth=5,
                   maxiterations=1, 
                   explorationweight=1.0,
                   earlystop=earlystop,
@@ -955,10 +952,10 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
   
   # compare all high value state answer then select the best one
   if length(highValueState) > 1
-    selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
+    selected = compareState(query, highValueState, text2textInstructLLM)
     resultState = highValueState[selected]
   end
-  
+
   max_ind =
     if length(resultState["action_history"]) == 0
       0
@@ -967,7 +964,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
       maximum(parse.(Int, k))
     end
   latest_action = resultState["action_history"]["$max_ind"]
-  
+  # println("\n")
+  # println(resultState)
+  # @info "---" @__LINE__
   #CHANGE add to vectorDB only if the answer is achieved and the state is terminal 
   sql = latest_action["action_input"]
   if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
@@ -975,9 +974,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
     insertSQLVectorDB(resultState["question"], sql)
   end
 
-  println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-  # pprintln(resultState)
-  println("---\n")
+  # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+  # # pprintln(resultState)
+  # println("---\n")
   
   return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"])
 end

src/llmfunction.jl

@@ -810,59 +810,71 @@ julia>
 - The LLM evaluates attempts based on accuracy and relevance to the original question
 """
 function compareState(question::String, highValueStateList::Vector{T},
-  text2textInstructLLM::Function, llmFormatName::String
+  text2textInstructLLM::Function; maxattempt::Integer=10
   )::Integer where {T<:AbstractDict}
 
   systemmsg = 
   """
-    # Your profile:
-    - You are a helpful assistant
-    # Situation:
-    - The user has made multiple attempts to solve the question, resulting in various answers
-    # Your mission:
-    - Identify and select the most accurate and relevant response from these multiple results for the user
-    # At each round of conversation, you will be given the following:
-      Question: the question the user is trying to answer
-      Attempt: the user's attempted actions and their corresponding results
-    # You should then respond to the user with the following:
-      Comparison: detailed comparison of all results from all attempts from various aspects.
-      Rationale: a brief explanation of why the selected response is the most accurate and relevant
-      Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
-    # You should only respond in format as described below:
-      Comparison: ...
-      Rationale: ...
-      Selected_response_number: ...
-    # Here are some examples:
-      User's question: "How many German wines do you have?"
-      Attempt 1)
-      action_name: RUNSQL
-      action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
-      action_result: 100 wines
-      Attempt 2)
-      action_name: RUNSQL
-      action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
-      action_result: 50 red wines
-      Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
-      Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
-      Selected_response_number:1
+  # Your profile:
+  - You are a helpful assistant
 
-    Let's begin!
+  # Situation:
+  - The user has made multiple attempts to solve the question, resulting in various answers
+  
+  # Your mission:
+  - Identify and select the most accurate and relevant response from these multiple results for the user
+  
+  # At each round of conversation, you will be given the following:
+    Question: the question the user is trying to answer
+    Attempt: the user's attempted actions and their corresponding results
+  
+  # You should then respond to the user with the following:
+    1) "comparison", detailed comparison of all results from all attempts from various aspects.
+    2) "rationale", a brief explanation of why the selected response is the most accurate and relevant
+    3) "selected_response_number", the number the selected response in the list of results (e.g., 1, 2, 3, ...)
+
+  # you should only respond in JSON format as described below
+    "comparison": "..."
+    "rationale": "..."
+    "selected_response_number": "..."
+
+  # Here are some examples:
+    Question: "How many German wines do you have?"
+    Attempt 1)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
+    action_result: 100 wines
+    Attempt 2)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
+    action_result: 50 red wines
+    
+    "comparison": "The second attempt counts only German red wines while the first attempt includes all German wines."
+    "rationale": "The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines."
+    "selected_response_number": "1"
   """
 
-potentialSolution = []
+  requiredKeys = ["comparison", "rationale", "selected_response_number"]
+
+  potentialSolution = []
    includekeys = ["action_name", "action_input", "action_result"]
    # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
    for state in highValueStateList
      action_history = state["action_history"]
-     latestKeys = [i for i in keys(d)][end]
+     latestKeys = [i for i in keys(action_history)][end]
      d = Dict()
      # get the last action_name, action_input, observation of currentstate
      for (i,v) in enumerate(includekeys)
-       d[v] = action_history[latestKeys[i]] #BUG ERROR: LoadError: KeyError: key "action_input_nothing" not found
+      latest_action = action_history[latestKeys]
+      d[v] = latest_action[v] 
      end
      push!(potentialSolution, d)
    end
 
+  println("\n")
+  @show potentialSolution
+  println("--- ", @__FILE__, @__LINE__)
+
   """
   # put potential solutions from potentialSolution into the following form
   Attempt 1)
@@ -875,73 +887,75 @@ potentialSolution = []
     action_result:
   ...
   """
+
   potentialSolutionStr = ""
   for (i, state) in enumerate(potentialSolution)
     potentialSolutionStr *= "Attempt $i)\n"
-    for k in keys
+    for k in includekeys
       potentialSolutionStr *= "$k: $(state[k])\n"
-      println("")
     end
   end
 
-  errornote = "N/A"
+  usermsg =
+  """
+  Question: $question
+  $potentialSolutionStr
+  """
 
-  for attempt in 1:10
-    errorFlag = false
+  msg = Dict(
+    "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
+    "messages" => [
+      Dict(
+        "role" => "system",
+        "content" => [
+          Dict("type" => "text", "text" => systemmsg),
+        ]
+      ),
+      Dict(
+        "role" => "user",
+        "content" => [        
+          Dict("type" => "text", "text" => usermsg),
+        ]
+      ),
+    ],
+    "temperature" => 0.7
+  )
 
-    usermsg =
-    """
-    Question: $question
-    Attempts: $potentialSolutionStr
-    P.S. $errornote
-    """
-
-    _prompt = 
-    [
-      Dict(:name=> "system", :text=> systemmsg),
-      Dict(:name=> "user", :text=> usermsg)
-    ]
-
-    # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
-
-    header = ["Comparison:", "Rationale:", "Selected_response_number:"]
-    dictkey = ["comparison", "rationale", "selected_response_number"]
-
-    response = text2textInstructLLM(prompt, modelsize="medium")
-
-    # sometime LLM output something like **Comprehension**: which is not expected
-    response = replace(response, "**"=>"")
-    response = replace(response, "***"=>"")
-    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+  for attempt in 1:maxattempt
+    response = text2textInstructLLM("random_id", msg)
+    response = GeneralUtils.clean_json_response(response)
+    response = GeneralUtils.remove_french_accents(response)
     think, response = GeneralUtils.extractthink(response)
-
-    # check whether response has all header
-    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
-    missingkeys = [k for (k, v) in detected_kw if v === nothing]
-    if !isempty(missingkeys)
-      errornote = "$missingkeys are missing from your previous response"
-      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue
-    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
-      errornote = "\nYour previous attempt has duplicated points according to the required response format"
-      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    response = String(split(response, ", action_result")[1])  # in case LLM generate action_result key which it isn't supposed to
+    response = strip(response)
+    responsedict = nothing
+    try
+      _responsedict = JSON.parse(response)
+      responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
+    catch
+      println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
-    responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=false)
+    # check whether all answer's key points are in responsedict
+    ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
+    if !ispass
+      errornote = errormsg
+      println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+      continue
+    end
 
-responsedict["selected_response_number"] = responsedict["selected_response_number"][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
-   try
-     responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
+    try
+      responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
     catch
       errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number."
       println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
-    println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-    pprintln(Dict(responsedict))
+    # println("\n~~~ compareState() ")
+    # pprintln(Dict(responsedict))
+    # println("---\n", @__FILE__, ":", @__LINE__)
 
     return responsedict["selected_response_number"]
   end