update

src/interface.jl

@@ -632,7 +632,7 @@ function evaluator(state::T1, text2textInstructLLM::Function;
 
   systemmsg = 
   """
-  You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions.
+  You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions.
 
   Definitions:
   "question" is the user's question
@@ -647,7 +647,7 @@ function evaluator(state::T1, text2textInstructLLM::Function;
 
   At each round of conversation, the user will give you:
   Context: ...
-  Trajectories: ...
+  Trajectory: ...
     
   You should then respond to the user with:
   1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
@@ -666,8 +666,11 @@ function evaluator(state::T1, text2textInstructLLM::Function;
       question: Find cars with a stereo.
       observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black.
   4) Score: Correctness score s where s is a single integer between 0 to 9. 
-      - 0 (the trajectories are incorrect.)
-      - 9 (the trajectories are correct, and the observation's content directly answers the question.)
+      Score guideline:
+      - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed
+      - 4 indicates that the trajectory are correct but the observation is incorrect or failed
+      - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
+      - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
   5) Suggestion: if accepted_as_answer is "No", provide suggestion.
 
   You should only respond in format as described below:
@@ -689,7 +692,7 @@ function evaluator(state::T1, text2textInstructLLM::Function;
   for attempt in 1:5
     usermsg =
     """
-    Trajectories: $thoughthistory
+    Trajectory: $thoughthistory
     $noise
     """
 
@@ -722,7 +725,7 @@ function evaluator(state::T1, text2textInstructLLM::Function;
       accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
       suggestion::AbstractString = responsedict[:suggestion]
 
-      if accepted_as_answer ∉ ["Yes", "No"]
+      if accepted_as_answer ∉ ["Yes", "No"] # [WORKING] add errornote into the prompt
         error("generated accepted_as_answer has wrong format")
       end
 
@@ -1277,7 +1280,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
                         ["Understanding", "Q1"], 
                         rightmarker=":", symbolkey=true; lowercasekey=true)
       response = "Q1: " * responsedict[:q1]
-      println("--> SQLLLM generatequestion ", @__FILE__, " ", @__LINE__)
+      println("\n~~~ SQLLLM generatequestion() ", @__FILE__, " ", @__LINE__)
       pprintln(Dict(responsedict))
       return response
     catch e