update

Reviewed-on: #5

Project.toml

@@ -1,6 +1,6 @@
 name = "SQLLLM"
 uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
-version = "0.2.7"
+version = "0.2.8"
 authors = ["narawat lamaiin <narawat@outlook.com>"]
 
 [deps]

etc.jl

@@ -0,0 +1,22 @@
+d = OrderedDict{String, Any}("1" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT w.winery, w.wine_name, w.wine_id, w.vintage, w.region, w.country, w.wine_type, w.grape, w.serving_temperature, w.sweetness, w.intensity, w.tannin, w.acidity, w.tasting_notes, rw.price, rw.currency, w.image_url FROM wine w JOIN retailer_wine rw ON w.wine_id = rw.wine_id JOIN retailer r ON rw.retailer_id = r.retailer_id WHERE w.wine_name ILIKE 'Montrachet Grand Cru' AND w.winery ILIKE 'Domaine Jacques Prieur' AND w.region ILIKE 'Montrachet' AND r.retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "No records found. Try loosening your search criteria.", "plan" => "The user wants to retrieve specific details about a wine, including its characteristics and pricing at a specific retailer, based on several criteria: wine name, winery, region, retailer name, and retailer ID. I need to join the 'wine', 'retailer_wine', and 'retailer' tables. I will use the wine attributes (name, winery, region) and the retailer ID to filter the results and select all requested fields, using ILIKE for text matching as per guidelines."), "2" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT retailer_name FROM retailer WHERE retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "1) retailer_name: Yiem Wines Ltd\n", "plan" => "The previous query failed with no records found, indicating that the combination of all constraints (wine name, winery, region, and retailer ID) is too restrictive. To proceed, I will first verify the retailer's name using the provided ID. Then, I will execute a new query, joining the 'wine', 'retailer_wine', and 'retailer' tables, filtering only by the wine name ('Montrachet Grand Cru') and the specific retailer ID ('f54eab6b-7650-4448-b009-c53f3efbcc3b'), removing the 'winery' and 'region' constraints to see if any matching records exist at that retailer."))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/interface.jl

@@ -103,7 +103,7 @@ Dict(
 # Signature
 """
 function decisionMaker(state::T1, text2textInstructLLM::Function, llmFormatName::String
-  ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
+  ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt::Integer=10
   )::Dict{String, Any} where {T1<:AbstractDict, T2<:Function}
 
   requiredKeys = ["plan", "action_name", "action_input"]
@@ -229,30 +229,30 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
 
     systemmsg =
   """
-  <situation>
+  # situation
     At each round of conversation, the user provides the following:
     - customer question
     - trajectory: A history of how an agent (you) worked on the question chronologically
-  </situation>
-  <objective>
+
+  # objective
     Analyze and evaluate agent's trajectory to find solutions and the results of actions to answer the user's questions according to evaluation guidelines.
-  </objective>
-  <your responsibility includes>
+
+  # your responsibility includes
     Fulfill the objective.
-  </your responsibility includes>
-  <evaluation guidelines>
+
+  # evaluation guidelines
     - When the search returns no result, it usually means 1) there is simply no data. or 2) SQL condition is not correct or 3) SQL is looking at the wrong tables.
     - validate whether the SQL query makes sense before accepting it as a valid answer.
-  </evaluation guidelines> 
-  <you should then respond to the user with>
-    1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
+
+  # you should then respond to the user with
+    1) "trajectory_evaluation", Analyze the trajectory of a solution to answer the user's original question.
       - Evaluate the correctness of each section and the overall trajectory based on the given question.
       - Provide detailed reasoning and analysis, focusing on the latest plan, action_name, action_input, and action_result.
       - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached.
       - Do not generate additional thoughts or actions.
-    2) Answer_evaluation: 
+    2) "answer_evaluation", 
       - Focus only on the matter mentioned in the question and comprehensively analyze how the latest action_input is appropriate.
-    3) Accepted_as_answer: Decide whether the latest action_input is technically correct. Can be "yes" or "no"
+    3) "accepted_as_answer", Decide whether the latest action_input is technically correct. Can be "yes" or "no"
       Bad example:
         question: Find cars with 4 wheels.
         action_input: INSERT INTO employees 
@@ -261,7 +261,7 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
         question: Find cars with a sunroof.
         action_input: SELECT * FROM car_features 
                       WHERE has_sunroof = TRUE;
-    4) Score: Correctness score s where s is a single integer between 0 to 9. 
+    4) "score", Correctness score s where s is a single integer between 0 to 9. 
       For example:
       - 0 indicates that both the trajectory is incorrect, failed or errors and the action_result is incorrect or failed
       - 4 indicates that the trajectory are correct, but no results are returned.
@@ -270,14 +270,13 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
       - 8 indicates that both the trajectory are correct, and the action_result's content directly answers the question.
       - 9 indicates a perfect perfomance. Both the trajectory are correct, and the action_result's content directly answers the question, surpassing your expectations.
     5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
-  </you should then respond to the user with>
-  <you should only respond in JSON format as described below>
+
+  # you should only respond in JSON format as described below
     "trajectory_evaluation": "...",
     "answer_evaluation": "...",
     "accepted_as_answer": "...",
     "score": "...",
     "suggestion": "..."
-  </you should only respond in JSON format as described below>
   """ 
   requiredKeys = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
   errornote = ""
@@ -318,7 +317,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
   for attempt in 1:maxattempt
     response = text2textInstructLLM("random_id", msg)
     response = GeneralUtils.clean_json_response(response)
-
     response = GeneralUtils.remove_french_accents(response)
     think, response = GeneralUtils.extractthink(response)
     response = String(split(response, ", action_result")[1])  # in case LLM generate action_result key which it isn't supposed to
@@ -340,7 +338,6 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
       continue
     end
     
-    responsedict["score"] = responsedict["score"][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
     try
       responsedict["score"] = parse(Int, responsedict["score"]) # convert string "5" into integer 5
     catch
@@ -942,9 +939,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
   
   root, _, resultState, highValueState = 
     LLMMCTS.runMCTS(initialstate, transition, transitionargs;
-                  horizontalSampleExpansionPhase=1, 
-                  horizontalSampleSimulationPhase=1,
-                  maxSimulationDepth=1,
+                  horizontalSampleExpansionPhase=3, 
+                  horizontalSampleSimulationPhase=3,
+                  maxSimulationDepth=5,
                   maxiterations=1, 
                   explorationweight=1.0,
                   earlystop=earlystop,
@@ -955,7 +952,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
   
   # compare all high value state answer then select the best one
   if length(highValueState) > 1
-    selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
+    selected = compareState(query, highValueState, text2textInstructLLM)
     resultState = highValueState[selected]
   end
 
@@ -967,7 +964,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
       maximum(parse.(Int, k))
     end
   latest_action = resultState["action_history"]["$max_ind"]
-  
+  # println("\n")
+  # println(resultState)
+  # @info "---" @__LINE__
   #CHANGE add to vectorDB only if the answer is achieved and the state is terminal 
   sql = latest_action["action_input"]
   if insertSQLVectorDB !== nothing && resultState["isterminal"] == true &&
@@ -975,9 +974,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
     insertSQLVectorDB(resultState["question"], sql)
   end
 
-  println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-  # pprintln(resultState)
-  println("---\n")
+  # println("\n--- SQLLLM query() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+  # # pprintln(resultState)
+  # println("---\n")
   
   return (result_str=latest_action["action_result"], result_raw=resultState["result_raw"])
 end

src/llmfunction.jl

@@ -502,7 +502,7 @@ function SQLexecution(executeSQL::Function, sql::T
     tablesize = size(df)
     row, column = tablesize
     if row == 0
-      return (result_str="No records found.", result_raw=df, success=true, errormsg=nothing)
+      return (result_str="No records found. Try loosening your search criteria.", result_raw=nothing, success=true, errormsg=nothing)
     elseif column > 30
       return (result_str="There are more than 30 columns. Please be more specific.", result_raw=df, success=true, errormsg=nothing)
     else
@@ -810,138 +810,152 @@ julia>
 - The LLM evaluates attempts based on accuracy and relevance to the original question
 """
 function compareState(question::String, highValueStateList::Vector{T},
-  text2textInstructLLM::Function, llmFormatName::String
+  text2textInstructLLM::Function; maxattempt::Integer=10
   )::Integer where {T<:AbstractDict}
 
   systemmsg = 
   """
-    Your profile:
-    - You are a helpful assistant
-    Situation:
-    - The user has made multiple attempts to solve the question, resulting in various answers
-    Your mission:
-    - Identify and select the most accurate and relevant response from these multiple results for the user
-    At each round of conversation, you will be given the following:
-      Question: the question the user is trying to answer
-      Attempt: the user's attempted actions and their corresponding results
-    You should then respond to the user with the following:
-      Comparison: detailed comparison of all results from all attempts from various aspects.
-      Rationale: a brief explanation of why the selected response is the most accurate and relevant
-      Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
-    You should only respond in format as described below:
-      Comparison: ...
-      Rationale: ...
-      Selected_response_number: ...
-    Here are some examples:
-      User's question: "How many German wines do you have?"
-      Attempt 1)
-      Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
-      Result: 100 wines
-      Attempt 2)
-      Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
-      Result: 50 red wines
-      Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
-      Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
-      Selected_response_number:1
+  # Your profile:
+  - You are a helpful assistant
 
-    Let's begin!
+  # Situation:
+  - The user has made multiple attempts to solve the question, resulting in various answers
+  
+  # Your mission:
+  - Identify and select the most accurate and relevant response from these multiple results for the user
+  
+  # At each round of conversation, you will be given the following:
+    Question: the question the user is trying to answer
+    Attempt: the user's attempted actions and their corresponding results
+  
+  # You should then respond to the user with the following:
+    1) "comparison", detailed comparison of all results from all attempts from various aspects.
+    2) "rationale", a brief explanation of why the selected response is the most accurate and relevant
+    3) "selected_response_number", the number the selected response in the list of results (e.g., 1, 2, 3, ...)
+
+  # you should only respond in JSON format as described below
+    "comparison": "..."
+    "rationale": "..."
+    "selected_response_number": "..."
+
+  # Here are some examples:
+    Question: "How many German wines do you have?"
+    Attempt 1)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
+    action_result: 100 wines
+    Attempt 2)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
+    action_result: 50 red wines
+    
+    "comparison": "The second attempt counts only German red wines while the first attempt includes all German wines."
+    "rationale": "The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines."
+    "selected_response_number": "1"
   """
 
-potentialSolution = []
-   keys = ["action_input", "observation"]
+  requiredKeys = ["comparison", "rationale", "selected_response_number"]
+
+  potentialSolution = []
+   includekeys = ["action_name", "action_input", "action_result"]
    # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
    for state in highValueStateList
      action_history = state["action_history"]
-     _, currentstate_latestIndice = 
-       GeneralUtils.findHighestIndexKey(action_history, keys[1])
-     latestKeys = makekey.(keys, currentstate_latestIndice)
+     latestKeys = [i for i in keys(action_history)][end]
      d = Dict()
      # get the last action_name, action_input, observation of currentstate
-     for (i,v) in enumerate(keys)
-       d[v] = action_history[latestKeys[i]]
+     for (i,v) in enumerate(includekeys)
+      latest_action = action_history[latestKeys]
+      d[v] = latest_action[v] 
      end
      push!(potentialSolution, d)
    end
 
+  println("\n")
+  @show potentialSolution
+  println("--- ", @__FILE__, @__LINE__)
+
   """
   # put potential solutions from potentialSolution into the following form
   Attempt 1)
     action_name: 
     action_input:
-    observation:
+    action_result:
   Attempt 2)
-    action_name:` 
+    action_name:
     action_input:
-    observation:`
+    action_result:
   ...
   """
+
   potentialSolutionStr = ""
   for (i, state) in enumerate(potentialSolution)
     potentialSolutionStr *= "Attempt $i)\n"
-    for k in keys
+    for k in includekeys
       potentialSolutionStr *= "$k: $(state[k])\n"
-      println("")
     end
   end
 
-  errornote = "N/A"
+  usermsg =
+  """
+  Question: $question
+  $potentialSolutionStr
+  """
 
-  for attempt in 1:10
-    errorFlag = false
+  msg = Dict(
+    "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
+    "messages" => [
+      Dict(
+        "role" => "system",
+        "content" => [
+          Dict("type" => "text", "text" => systemmsg),
+        ]
+      ),
+      Dict(
+        "role" => "user",
+        "content" => [        
+          Dict("type" => "text", "text" => usermsg),
+        ]
+      ),
+    ],
+    "temperature" => 0.7
+  )
 
-    usermsg =
-    """
-    Question: $question
-    Attempts: $potentialSolutionStr
-    P.S. $errornote
-    """
-
-    _prompt = 
-    [
-      Dict(:name=> "system", :text=> systemmsg),
-      Dict(:name=> "user", :text=> usermsg)
-    ]
-
-    # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
-
-    header = ["Comparison:", "Rationale:", "Selected_response_number:"]
-    dictkey = ["comparison", "rationale", "selected_response_number"]
-
-    response = text2textInstructLLM(prompt, modelsize="medium")
-
-    # sometime LLM output something like **Comprehension**: which is not expected
-    response = replace(response, "**"=>"")
-    response = replace(response, "***"=>"")
-    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+  for attempt in 1:maxattempt
+    response = text2textInstructLLM("random_id", msg)
+    response = GeneralUtils.clean_json_response(response)
+    response = GeneralUtils.remove_french_accents(response)
     think, response = GeneralUtils.extractthink(response)
-
-    # check whether response has all header
-    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
-    missingkeys = [k for (k, v) in detected_kw if v === nothing]
-    if !isempty(missingkeys)
-      errornote = "$missingkeys are missing from your previous response"
-      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue
-    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
-      errornote = "\nYour previous attempt has duplicated points according to the required response format"
-      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    response = String(split(response, ", action_result")[1])  # in case LLM generate action_result key which it isn't supposed to
+    response = strip(response)
+    responsedict = nothing
+    try
+      _responsedict = JSON.parse(response)
+      responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
+    catch
+      println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
-    responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=false)
+    # check whether all answer's key points are in responsedict
+    ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
+    if !ispass
+      errornote = errormsg
+      println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+      continue
+    end
 
-responsedict["selected_response_number"] = responsedict["selected_response_number"][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
-   try
-     responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
+    try
+      responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
     catch
       errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number."
       println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
-    println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-    pprintln(Dict(responsedict))
+    # println("\n~~~ compareState() ")
+    # pprintln(Dict(responsedict))
+    # println("---\n", @__FILE__, ":", @__LINE__)
 
     return responsedict["selected_response_number"]
   end