2026-07-15 04:50:38 +00:00
3 changed files with 58 additions and 36 deletions
@@ -0,0 +1,22 @@
+d = OrderedDict{String, Any}("1" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT w.winery, w.wine_name, w.wine_id, w.vintage, w.region, w.country, w.wine_type, w.grape, w.serving_temperature, w.sweetness, w.intensity, w.tannin, w.acidity, w.tasting_notes, rw.price, rw.currency, w.image_url FROM wine w JOIN retailer_wine rw ON w.wine_id = rw.wine_id JOIN retailer r ON rw.retailer_id = r.retailer_id WHERE w.wine_name ILIKE 'Montrachet Grand Cru' AND w.winery ILIKE 'Domaine Jacques Prieur' AND w.region ILIKE 'Montrachet' AND r.retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "No records found. Try loosening your search criteria.", "plan" => "The user wants to retrieve specific details about a wine, including its characteristics and pricing at a specific retailer, based on several criteria: wine name, winery, region, retailer name, and retailer ID. I need to join the 'wine', 'retailer_wine', and 'retailer' tables. I will use the wine attributes (name, winery, region) and the retailer ID to filter the results and select all requested fields, using ILIKE for text matching as per guidelines."), "2" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT retailer_name FROM retailer WHERE retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "1) retailer_name: Yiem Wines Ltd\n", "plan" => "The previous query failed with no records found, indicating that the combination of all constraints (wine name, winery, region, and retailer ID) is too restrictive. To proceed, I will first verify the retailer's name using the provided ID. Then, I will execute a new query, joining the 'wine', 'retailer_wine', and 'retailer' tables, filtering only by the wine name ('Montrachet Grand Cru') and the specific retailer ID ('f54eab6b-7650-4448-b009-c53f3efbcc3b'), removing the 'winery' and 'region' constraints to see if any matching records exist at that retailer."))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -229,22 +229,22 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str

    systemmsg =
  """
-  <situation>
+  # situation
    At each round of conversation, the user provides the following:
    - customer question
    - trajectory: A history of how an agent (you) worked on the question chronologically
-  </situation>
-  <objective>
+
+  # objective
    Analyze and evaluate agent's trajectory to find solutions and the results of actions to answer the user's questions according to evaluation guidelines.
-  </objective>
-  <your responsibility includes>
+
+  # your responsibility includes
    Fulfill the objective.
-  </your responsibility includes>
-  <evaluation guidelines>
+
+  # evaluation guidelines
    - When the search returns no result, it usually means 1) there is simply no data. or 2) SQL condition is not correct or 3) SQL is looking at the wrong tables.
    - validate whether the SQL query makes sense before accepting it as a valid answer.
-  </evaluation guidelines> 
-  <you should then respond to the user with>
+
+  # you should then respond to the user with
    1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
      - Evaluate the correctness of each section and the overall trajectory based on the given question.
      - Provide detailed reasoning and analysis, focusing on the latest plan, action_name, action_input, and action_result.
@@ -270,14 +270,14 @@ function evaluator(state::T1, text2textInstructLLM::Function, llmFormatName::Str
      - 8 indicates that both the trajectory are correct, and the action_result's content directly answers the question.
      - 9 indicates a perfect perfomance. Both the trajectory are correct, and the action_result's content directly answers the question, surpassing your expectations.
    5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
-  </you should then respond to the user with>
-  <you should only respond in JSON format as described below>
+
+  # you should only respond in JSON format as described below
    "trajectory_evaluation": "...",
    "answer_evaluation": "...",
    "accepted_as_answer": "...",
    "score": "...",
    "suggestion": "..."
-  </you should only respond in JSON format as described below>
+
  """ 
  requiredKeys = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
  errornote = ""
@@ -942,9 +942,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
  
  root, _, resultState, highValueState = 
    LLMMCTS.runMCTS(initialstate, transition, transitionargs;
-                  horizontalSampleExpansionPhase=1, 
-                  horizontalSampleSimulationPhase=1,
-                  maxSimulationDepth=1,
+                  horizontalSampleExpansionPhase=3, 
+                  horizontalSampleSimulationPhase=3,
+                  maxSimulationDepth=3,
                  maxiterations=1, 
                  explorationweight=1.0,
                  earlystop=earlystop,
@@ -958,7 +958,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
    selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
    resultState = highValueState[selected]
  end
-
+  
  max_ind =
    if length(resultState["action_history"]) == 0
      0
@@ -815,31 +815,33 @@ function compareState(question::String, highValueStateList::Vector{T},

  systemmsg = 
  """
-    Your profile:
+    # Your profile:
    - You are a helpful assistant
-    Situation:
+    # Situation:
    - The user has made multiple attempts to solve the question, resulting in various answers
-    Your mission:
+    # Your mission:
    - Identify and select the most accurate and relevant response from these multiple results for the user
-    At each round of conversation, you will be given the following:
+    # At each round of conversation, you will be given the following:
      Question: the question the user is trying to answer
      Attempt: the user's attempted actions and their corresponding results
-    You should then respond to the user with the following:
+    # You should then respond to the user with the following:
      Comparison: detailed comparison of all results from all attempts from various aspects.
      Rationale: a brief explanation of why the selected response is the most accurate and relevant
      Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
-    You should only respond in format as described below:
+    # You should only respond in format as described below:
      Comparison: ...
      Rationale: ...
      Selected_response_number: ...
-    Here are some examples:
+    # Here are some examples:
      User's question: "How many German wines do you have?"
      Attempt 1)
-      Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
-      Result: 100 wines
+      action_name: RUNSQL
+      action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
+      action_result: 100 wines
      Attempt 2)
-      Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
-      Result: 50 red wines
+      action_name: RUNSQL
+      action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
+      action_result: 50 red wines
      Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
      Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
      Selected_response_number:1
@@ -848,17 +850,15 @@ function compareState(question::String, highValueStateList::Vector{T},
  """

 potentialSolution = []
-   keys = ["action_input", "observation"]
+   includekeys = ["action_name", "action_input", "action_result"]
   # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
   for state in highValueStateList
     action_history = state["action_history"]
-     _, currentstate_latestIndice = 
-       GeneralUtils.findHighestIndexKey(action_history, keys[1])
-     latestKeys = makekey.(keys, currentstate_latestIndice)
+     latestKeys = [i for i in keys(d)][end]
     d = Dict()
     # get the last action_name, action_input, observation of currentstate
-     for (i,v) in enumerate(keys)
-       d[v] = action_history[latestKeys[i]]
+     for (i,v) in enumerate(includekeys)
+       d[v] = action_history[latestKeys[i]] #BUG ERROR: LoadError: KeyError: key "action_input_nothing" not found
     end
     push!(potentialSolution, d)
   end
@@ -868,11 +868,11 @@ potentialSolution = []
  Attempt 1)
    action_name: 
    action_input:
-    observation:
+    action_result:
  Attempt 2)
-    action_name:` 
+    action_name:
    action_input:
-    observation:`
+    action_result:
  ...
  """
  potentialSolutionStr = ""
				`@@ -0,0 +1,22 @@`
				d = OrderedDict{String, Any}("1" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT w.winery, w.wine_name, w.wine_id, w.vintage, w.region, w.country, w.wine_type, w.grape, w.serving_temperature, w.sweetness, w.intensity, w.tannin, w.acidity, w.tasting_notes, rw.price, rw.currency, w.image_url FROM wine w JOIN retailer_wine rw ON w.wine_id = rw.wine_id JOIN retailer r ON rw.retailer_id = r.retailer_id WHERE w.wine_name ILIKE 'Montrachet Grand Cru' AND w.winery ILIKE 'Domaine Jacques Prieur' AND w.region ILIKE 'Montrachet' AND r.retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "No records found. Try loosening your search criteria.", "plan" => "The user wants to retrieve specific details about a wine, including its characteristics and pricing at a specific retailer, based on several criteria: wine name, winery, region, retailer name, and retailer ID. I need to join the 'wine', 'retailer_wine', and 'retailer' tables. I will use the wine attributes (name, winery, region) and the retailer ID to filter the results and select all requested fields, using ILIKE for text matching as per guidelines."), "2" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT retailer_name FROM retailer WHERE retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "1) retailer_name: Yiem Wines Ltd\n", "plan" => "The previous query failed with no records found, indicating that the combination of all constraints (wine name, winery, region, and retailer ID) is too restrictive. To proceed, I will first verify the retailer's name using the provided ID. Then, I will execute a new query, joining the 'wine', 'retailer_wine', and 'retailer' tables, filtering only by the wine name ('Montrachet Grand Cru') and the specific retailer ID ('f54eab6b-7650-4448-b009-c53f3efbcc3b'), removing the 'winery' and 'region' constraints to see if any matching records exist at that retailer."))