update

src/interface.jl

@@ -102,8 +102,8 @@ Dict(
 
 # Signature
 """
-function decisionMaker(state::T1, context, text2textInstructLLM::Function,
-  ; querySQLVectorDBF::Union{T2, Nothing}=nothing
+function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function, llmFormatName::String
+  ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
   )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:Function}
 
   # lessonDict =
@@ -135,19 +135,14 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
 
   systemmsg = 
   """
-    You are a helpful assistant that find the data from a database to satisfy the user's query. 
-    You are also eager to improve your helpfulness.
+    You are a helpful assistant that find the data from a database to satisfy the user's question. 
+    You are working under your mentor supervision and you are also eager to improve your helpfulness.
 
     For your information:
     - Observation: Result of the immediately preceding action
 
-    At each round of conversation, the user will give you the following:
-    User Query: ...
-    Example: ...
-    Your Q&A: ...
-    Your work progress: ...
-    Evaluation: Evaluation of the immediately preceding action and observation
-    Suggestion: Suggestion for the immediately preceding action and observation
+    At each round of conversation, you will be given the following information:
+    context: additional information about the current situation
     
     You must follow the following guidelines:
     - Keep SQL queries focused only on the provided information.
@@ -156,28 +151,28 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
     - Do not create any table in the database
     - A junction table can be used to link tables together. Another use case is for filtering data.
     - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
-    - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
     - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
+    - If there is no search result from the database, remove the restrictive criteria until a search result is available, and proceed from there.
 
     You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input:
-      Comprehension: state your comprehension about the current situation.
-      Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
-      Action_name: (Typically corresponds to the execution of the first step in your plan)
+      1) plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+      2) action_name: (Typically corresponds to the execution of the first step in your plan)
         Can be one of the following function names: 
         - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
           For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
           Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
-    4) Action_input: Input to the action
+      3) action_input: Input to the action
 
-    You should only respond in format as described below:
-    Comprehension: ...
-    Plan: ...
-    Action_name: ...
-    Action_input: ...
+    You should only respond in JSON format as described below:
+    {
+      "plan": "...",
+      "action_name": "...",
+      "action_input": "..."
+    }
 
     Let's begin!
   """
-  
+  requiredKeys = [:plan, :action_name, :action_input]
   workprogress = ""
   for (k, v) in state[:thoughtHistory]
     if k ∉ [:question]
@@ -195,105 +190,77 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
     similarSQL_ = sql !== nothing ? sql : "None"
   end
 
-  header = ["Comprehension:", "Plan:", "Action_name:", "Action_input:"]
-  dictkey = ["comprehension", "plan", "action_name", "action_input"]
 
-  llmkwargs=Dict(
-      :num_ctx => 32768,
-      :temperature => 0.1,
-    )
+  for attempt in 1:maxattempt
 
-  for attempt in 1:10
-    if attempt > 1
-      println("\nERROR SQLLLM decisionMaker() attempt $attempt/10 ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      llmkwargs[:temperature] = 0.1 * attempt
-    end
+    # QandA = generatequestion(state, context, text2textInstructLLM, llmFormatName; similarSQL=similarSQL_)
 
-    QandA = generatequestion(state, context, text2textInstructLLM; similarSQL=similarSQL_)
-
-    usermsg =
+    context =
     """
-    $(context[:tablelist])
-    User query: $(state[:thoughtHistory][:question])
-    Example: $similarSQL_
-    Your Q&A: $QandA
-    Your work progress: $workprogress
-    Evaluation: $(state[:evaluation])
-    Suggestion: $(state[:suggestion])
-    P.S. $errornote
+    <context>
+      <table_schema> This is schema of tables in the database:
+      $(additionalinfo[:tablelist])
+      </table_schema>
+      <most_relevant_SQL> The closest known SQL for this question is:
+      $similarSQL_
+      </most_relevant_SQL>
+      <query_result_of_most_relevant_SQL> This is the query result when executing the most_relevant_SQL against a database. You can use this to see how the data are stored.
+      winery: Chateau Montelena, wine_name: The Montelena Estate Cabernet Sauvignon, wine_id: 97264f71-007c-4cce-a3fe-2cc88fba4d05, vintage: 2017, region: Napa Valley, country: United States, wine_type: red, grape: Cabernet Sauvignon, serving_temperature: 15 to 18 Celsius, sweetness: 1, intensity: 5, tannin: 4, acidity: 4, tasting_notes: oak, vanilla, tobacco, blackberry, plum, black cherry, leather, earthy, smoke, price: 19.95, currency: USD
+      </query_result_of_most_relevant_SQL>
+      <progress> your work progress so far:
+      $workprogress
+      </progress>
+      <suggestion> This is your mentor's suggestion for the immediately preceding action and observation
+      $(state[:suggestion])
+      </suggestion>
+      P.S. $errornote
+    </context>
     """
 
-    _prompt = 
-    [
-      Dict(:name=> "system", :text=> systemmsg),
-      Dict(:name=> "user", :text=> usermsg)
-    ]
+    unformatPrompt =
+      [
+        Dict(:name => "system", :text => systemmsg),
+        Dict(:name => "user", :text => state[:thoughtHistory][:question])
+      ]
 
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
-    response = text2textInstructLLM(prompt; llmkwargs=llmkwargs)
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
+    prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName)
+    # add info
+    prompt = prompt * context
+    response = text2textInstructLLM(prompt)
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
 
-    # LLM tends to generate observation given that it is in the input
-    response = 
-      if occursin("observation:", response)
-        string(split(response, "observation:")[1])
-      elseif occursin("Observation:", response)
-        string(split(response, "Observation:")[1])
-      elseif occursin("observation_", response)
-        string(split(response, "observation_")[1])
-      elseif occursin("Observation_", response)
-        string(split(response, "Observation_")[1])
-      else
-        response
-      end
-
-    # sometime LLM output something like **Comprehension**: which is not expected
-    response = replace(response, "**"=>"")
-    response = replace(response, "***"=>"")
-
-    # some time LLM output Plan_1: so we need to detect and replace topic numbering
-    regex = r"_[0-1000]+:"
-    matches = collect(eachmatch(regex, response))
-    for m in matches
-      response = replace(response, string(m.match)=>":") 
-    end
-
-    if occursin("NULL", response)
-      errornote = "\nYour previous attempt was NULL. This is not allowed"
-      println("\nERROR SQLLLM decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue
-    end
-
-    # # detect if there are more than 1 key per categories
-    # wordcount = GeneralUtils.countGivenWords(response, header)
-    # duplicateKeywordFlag = false
-    # for (i, v) in enumerate(wordcount)
-    #   keyword = header[i]
-    #   keywordNumber = v
-    #   if keywordNumber > 1
-    #     errornote = "\nSQL query has duplicated keyword, $keyword"
-    #     println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-    #     duplicateKeywordFlag = true
-    #     break
-    #   end
+    # if occursin("NULL", response)
+    #   errornote = "\nYour previous attempt contain NULL. It is not allowed in your response"
+    #   println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> \n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    #   continue
     # end
-    # duplicateKeywordFlag == true ? continue : nothing
 
-    # check whether response has all header
-    detected_kw = GeneralUtils.detect_keyword(header, response)
-    if 0 ∈ values(detected_kw)
-      errornote = "\nYour previous attempt did not have all points according to the required response format"
-      println("\nERROR SQLLLM decisionMaker() $errornote \n$response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue
-    elseif sum(values(detected_kw)) > length(header)
-      errornote = "\nYour previous attempt has duplicated points according to the required response format"
-      println("\nERROR SQLLLM decisionMaker() $errornote \n$response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    responsedict = nothing
+    try
+      responsedict = copy(JSON3.read(response))  
+    catch
+      println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
-    responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
+    # check whether all answer's key points are in responsedict
+    _responsedictKey = keys(responsedict)
+    responsedictKey = [i for i in _responsedictKey] # convert into a list
+    is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys]
+
+    if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys)
+      errornote = "Your previous attempt has more key points than answer's required key points."
+      println("\nERROR YiemAgent generatechat() $errornote --(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    elseif !all(is_requiredKeys_in_responsedictKey)
+      zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey)
+      missingkeys = [requiredKeys[i] for i in zeroind]
+      errornote = "$missingkeys are missing from your previous response"
+      println("\nERROR YiemAgent generatechat() $errornote --(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    end
 
     delete!(responsedict, :observation)
 
@@ -311,35 +278,296 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
 
     toollist = ["TABLEINFO", "RUNSQL"]
     if responsedict[:action_name] ∉ toollist
-      errornote = "\nYour previous attempt has action_name that is not in the tool list"
-      println("\nERROR SQLLLM decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      errornote = "Your previous attempt has action_name that is not in the tool list"
+      println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict[:action_name]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
     for i in toollist
       if occursin(i, responsedict[:action_input])
-        errornote = "\nYour previous attempt has action_name in action_input which is not allowed"
-        println("\nERROR SQLLLM decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+        errornote = "Your previous attempt has action_name in action_input which is not allowed"
+        println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict[:action_input]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
         continue
       end
     end
 
-    for i ∈ Symbol.(dictkey)
-      if length(JSON3.write(responsedict[i])) == 0
-        errornote = "\nYour previous attempt has empty value for $i"
-        println("\nERROR SQLLLM decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-        continue
-      end
-    end
+    # for i ∈ Symbol.(dictkey)
+    #   if length(JSON3.write(responsedict[i])) == 0
+    #     errornote = "Your previous attempt has empty value for $i"
+    #     println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    #     continue
+    #   end
+    # end
 
-    state[:decisionMaker] = responsedict
+    println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    pprintln(Dict(responsedict))
+
+    # store for later training
+    responsedict[:thoughthistory] = state[:thoughtHistory]
+    responsedict[:system] = systemmsg
+    responsedict[:prompt] = prompt
+    responsedict[:context] = context
+    responsedict[:think] = think
+
+    # # read sessionId
+    # sessionid = JSON3.read("/appfolder/app/sessionid.json")
+    # # save to filename ./log/decisionlog.txt
+    # println("saving SQLLLM decisionMaker() to disk")
+    # filename = "agent_decision_log_$(sessionid[:id]).json"
+    # filepath = "/appfolder/app/log/$filename"
+    # # check whether there is a file path exists before writing to it
+    # if !isfile(filepath)
+    #   decisionlist = [responsedict]
+    #   println("Creating file $filepath")
+    #   open(filepath, "w") do io
+    #     JSON3.pretty(io, decisionlist)
+    #   end
+    # else
+    #   # read the file and append new data
+    #   decisionlist = copy(JSON3.read(filepath))
+    #   push!(decisionlist, responsedict)
+    #   println("Appending new data to file $filepath")
+    #   open(filepath, "w") do io
+    #     JSON3.pretty(io, decisionlist)
+    #   end
+    # end
 
     return responsedict
-
   end
   error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
 end
 
+# function decisionMaker(state::T1, context, text2textInstructLLM::Function, llmFormatName::String
+#   ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
+#   )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:Function}
+
+#   # lessonDict =
+#   # if isfile("lesson.json")
+#   #   lessonDict = copy(JSON3.read("lesson.json"))
+#   # else
+#   #   lessonDict = nothing
+#   # end
+
+#   # lessonDict = nothing
+
+#   # lesson =
+#   # if lessonDict === nothing
+#   #   ""
+#   # else
+#   #   """
+#   #   You have attempted to help the user before and failed, either because your reasoning for the 
+#   #   recommendation was incorrect or your response did not exactly match the user expectation. 
+#   #   The following lesson(s) give a plan to avoid failing to help the user in the same way you 
+#   #   did previously. Use them to improve your strategy to help the user.
+
+#   #   Here are some lessons in JSON format:
+#   #   $(JSON3.write(lessonDict))
+
+#   #   When providing the thought and action for the current trial, that into account these failed 
+#   #   trajectories and make sure not to repeat the same mistakes and incorrect answers. 
+#   #   """
+#   # end
+
+#   systemmsg = 
+#   """
+#     You are a helpful assistant that find the data from a database to satisfy the user's query. 
+#     You are also eager to improve your helpfulness.
+
+#     For your information:
+#     - Observation: Result of the immediately preceding action
+
+#     At each round of conversation, the user will give you the following:
+#     User Query: ...
+#     Example: ...
+#     Your Q&A: ...
+#     Your work progress: ...
+#     Evaluation: Evaluation of the immediately preceding action and observation
+#     Suggestion: Suggestion for the immediately preceding action and observation
+    
+#     You must follow the following guidelines:
+#     - Keep SQL queries focused only on the provided information.
+
+#     You should follow the following guidelines:
+#     - Do not create any table in the database
+#     - A junction table can be used to link tables together. Another use case is for filtering data.
+#     - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
+#     - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
+#     - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
+
+#     You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input:
+#       Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+#       Action_name: (Typically corresponds to the execution of the first step in your plan)
+#         Can be one of the following function names: 
+#         - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
+#           For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
+#           Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
+#     4) Action_input: Input to the action
+
+#     You should only respond in format as described below:
+#     Plan: ...
+#     Action_name: ...
+#     Action_input: ...
+
+#     Let's begin!
+#   """
+  
+#   workprogress = ""
+#   for (k, v) in state[:thoughtHistory]
+#     if k ∉ [:question]
+#       workprogress *= "$k: $v\n"
+#     end
+#   end
+  
+#   response = nothing # store for show when error msg show up
+#   errornote = "N/A"
+
+#   # provide similar sql only for the first attempt
+#   similarSQL_ = "None"
+#   if length(state[:thoughtHistory]) == 1
+#     sql, distance = querySQLVectorDBF(state[:thoughtHistory][:question])
+#     similarSQL_ = sql !== nothing ? sql : "None"
+#   end
+
+#   header = ["Plan:", "Action_name:", "Action_input:"]
+#   dictkey = ["plan", "action_name", "action_input"]
+
+#   llmkwargs=Dict(
+#       :num_ctx => 32768,
+#       :temperature => 0.5,
+#     )
+
+#   for attempt in 1:maxattempt
+
+#     QandA = generatequestion(state, context, text2textInstructLLM, llmFormatName; similarSQL=similarSQL_)
+
+#     usermsg =
+#     """
+#     $(context[:tablelist])
+#     User query: $(state[:thoughtHistory][:question])
+#     Example: $similarSQL_
+#     Your Q&A: $QandA
+#     Your work progress: $workprogress
+#     Evaluation: $(state[:evaluation])
+#     Suggestion: $(state[:suggestion])
+#     P.S. $errornote
+#     """
+
+#     _prompt = 
+#     [
+#       Dict(:name=> "system", :text=> systemmsg),
+#       Dict(:name=> "user", :text=> usermsg)
+#     ]
+
+#     # put in model format
+#     prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
+#     response = text2textInstructLLM(prompt; llmkwargs=llmkwargs)
+#     response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+#     think, response = GeneralUtils.extractthink(response)
+
+#     # LLM tends to generate observation given that it is in the input
+#     response = 
+#       if occursin("observation:", response)
+#         string(split(response, "observation:")[1])
+#       elseif occursin("Observation:", response)
+#         string(split(response, "Observation:")[1])
+#       elseif occursin("observation_", response)
+#         string(split(response, "observation_")[1])
+#       elseif occursin("Observation_", response)
+#         string(split(response, "Observation_")[1])
+#       else
+#         response
+#       end
+
+#     # sometime LLM output something like **Comprehension**: which is not expected
+#     response = replace(response, "**"=>"")
+#     response = replace(response, "***"=>"")
+
+#     # some time LLM output Plan_1: so we need to detect and replace topic numbering
+#     regex = r"_[0-1000]+:"
+#     matches = collect(eachmatch(regex, response))
+#     for m in matches
+#       response = replace(response, string(m.match)=>":") 
+#     end
+
+#     if occursin("NULL", response)
+#       errornote = "\nYour previous attempt was NULL. This is not allowed"
+#       println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue
+#     end
+
+#     # # detect if there are more than 1 key per categories
+#     # wordcount = GeneralUtils.countGivenWords(response, header)
+#     # duplicateKeywordFlag = false
+#     # for (i, v) in enumerate(wordcount)
+#     #   keyword = header[i]
+#     #   keywordNumber = v
+#     #   if keywordNumber > 1
+#     #     errornote = "\nSQL query has duplicated keyword, $keyword"
+#     #     println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#     #     duplicateKeywordFlag = true
+#     #     break
+#     #   end
+#     # end
+#     # duplicateKeywordFlag == true ? continue : nothing
+
+#     # check whether response has all header
+#     detected_kw = GeneralUtils.detect_keyword(header, response)
+#     if 0 ∈ values(detected_kw)
+#       errornote = "\nYour previous attempt did not have all points according to the required response format"
+#       println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue
+#     elseif sum(values(detected_kw)) > length(header)
+#       errornote = "\nYour previous attempt has duplicated points according to the required response format"
+#       println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue
+#     end
+
+#     responsedict = GeneralUtils.textToDict(response, header; 
+#                                             dictKey=dictkey, symbolkey=true)
+
+#     delete!(responsedict, :observation)
+
+#     # remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String})
+#     if occursin("```", responsedict[:action_input])
+#       sql = GeneralUtils.extract_triple_backtick_text(responsedict[:action_input])[1]
+#       if sql[1:4] == "sql\n"
+#         sql = sql[5:end]
+#       end
+#       sql = split(sql, ';')   # some time there are comments in the sql
+#       sql = sql[1] * ';'
+
+#       responsedict[:action_input] = sql
+#     end
+
+#     toollist = ["TABLEINFO", "RUNSQL"]
+#     if responsedict[:action_name] ∉ toollist
+#       errornote = "Your previous attempt has action_name that is not in the tool list"
+#       println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue
+#     end
+
+#     for i in toollist
+#       if occursin(i, responsedict[:action_input])
+#         errornote = "Your previous attempt has action_name in action_input which is not allowed"
+#         println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#         continue
+#       end
+#     end
+
+#     for i ∈ Symbol.(dictkey)
+#       if length(JSON3.write(responsedict[i])) == 0
+#         errornote = "Your previous attempt has empty value for $i"
+#         println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#         continue
+#       end
+#     end
+
+#     state[:decisionMaker] = responsedict
+#     return responsedict
+#   end
+#   error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
+# end
+
 
 """ Assigns a scalar value to each new child node to be used for selec-
 tion and backpropagation. This value effectively quantifies the agent's progress in task completion,
@@ -361,7 +589,8 @@ julia>
 
 # Signature
 """
-function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
+function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String; 
+  maxattempt=10
   ) where {T1<:AbstractDict}
 
   systemmsg = 
@@ -370,8 +599,6 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
 
   Definitions:
   "question" is the user's question
-  "understanding" is agent's understanding about the current situation
-  "reasoning" is agent's step-by-step reasoning about the current situation
   "plan" is agent's plan to complete the task from the current situation
   "action_name" is the name of the action taken, which can be one of the following functions:
       - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
@@ -380,16 +607,14 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
   "action_input" is the input to the action
   "observation" is result of the preceding immediate action
 
-  <At each round of conversation, the user will give you>
-  Trajectory: ...
-  Error_note: error note from your previous attempt
-  </At each round of conversation, the user will give you>
+  At each round of conversation, you will be given the following information:
+  trajectory: A history of how you worked on the question chronologically
+  evaluatee_context: The context that evaluatee use to make a decision
 
-  <You must follow the following guidelines>
+  You must follow the following guidelines:
   - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
-  </You must follow the following guidelines>  
 
-  <You should then respond to the user with>
+  You should then respond to the user with:
   1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
     - Evaluate the correctness of each section and the overall trajectory based on the given question.
     - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation.
@@ -397,7 +622,7 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
     - Do not generate additional thoughts or actions.
   2) Answer_evaluation: 
     - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
-  3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No"
+  3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "yes" or "no"
     Bad example (The observation didn't answers the question):
       question: Find cars with 4 wheels.
       observation: There are an apple in the table.
@@ -407,66 +632,77 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
   4) Score: Correctness score s where s is a single integer between 0 to 9. 
       For example:
       - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed
-      - 4 indicates that the trajectory are correct but the observation is incorrect or failed
-      - 5 indicates that the trajectory are correct, but no results are returned.
+      - 4 indicates that the trajectory are correct, but no results are returned.
+      - 5 indicates that the trajectory are correct but the observation is incorrect or failed
       - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question
       - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question.
       - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations.
-  5) Suggestion: if accepted_as_answer is "No", provide suggestion.
-  </You should then respond to the user with>
+  5) Suggestion: what are the possible reason of this outcome, what can one learn from it and what suggestion can made?
 
-  <You should only respond in format as described below>
+  You should only respond in format as described below:
   Trajectory_evaluation: ...
   Answer_evaluation: ...
   Accepted_as_answer: ...
   Score: ...
   Suggestion: ...
-  </You should only respond in format as described below>
 
   Let's begin!
   """
+  #[WORKING] add what I should think --> this will be the think for decisionMaker()
+  header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"]
+  dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
 
   thoughthistory = ""
   for (k, v) in state[:thoughtHistory]
     thoughthistory *= "$k: $v\n" 
   end
 
-  errornote = ""
+  errornote = "N/A"
   for attempt in 1:maxattempt
-    usermsg =
-    """
-    Trajectory: $thoughthistory
-    P.S. $errornote
+      usermsg =
+      """
+      <trajectory>
+      $thoughthistory
+      </trajectory>
+      """
+    context =
+      """
+      <context>
+        <evaluatee_context>
+        thoughtDict[:context]
+        </evaluatee_context>
+        P.S. $errornote
+      </context>
     """
 
-    _prompt = 
-    [
-      Dict(:name=> "system", :text=> systemmsg),
-      Dict(:name=> "user", :text=> usermsg)
-    ]
-
+    unformatPrompt =
+      [
+        Dict(:name => "system", :text => systemmsg),
+        Dict(:name => "user", :text => usermsg)
+      ]
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
-
-    header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"]
-    dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
+    prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName)
+    # add info
+    prompt = prompt * context
 
     response = text2textInstructLLM(prompt, modelsize="medium")
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
 
     # sometime LLM output something like **Comprehension**: which is not expected
     response = replace(response, "**"=>"")
     response = replace(response, "***"=>"")
 
     # check whether response has all header
-    detected_kw = GeneralUtils.detect_keyword(header, response)
-    if 0 ∈ values(detected_kw)
-      errornote = "Your previous attempt does not have all answer points"
-      println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
+    missingkeys = [k for (k, v) in detected_kw if v === nothing]
+    if !isempty(missingkeys)
+      errornote = "$missingkeys are missing from your previous response"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
-    elseif sum(values(detected_kw)) > length(header)
-      errornote = "Your previous attempt has duplicated answer point"
-      println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
+      errornote = "\nYour previous attempt has duplicated points according to the required response format"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
@@ -484,9 +720,9 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
 
     accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
 
-    if accepted_as_answer ∉ ["Yes", "No"] # [PENDING] add errornote into the prompt
+    if accepted_as_answer ∉ ["yes", "no"]
       errornote = "Your previous attempt's accepted_as_answer has wrong format"
-      println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      println("\nERROR SQLLLM evaluator() Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict[:accepted_as_answer]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
@@ -497,7 +733,7 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
     state[:suggestion] = responsedict[:suggestion]
 
     # mark as terminal state when the answer is achieved
-    if accepted_as_answer == "Yes"
+    if accepted_as_answer ∈ ["Yes", "yes"] 
 
       # mark the state as terminal state because the evaluation say so.
       state[:isterminal] = true
@@ -505,9 +741,41 @@ function evaluator(state::T1, text2textInstructLLM::Function; maxattempt=10
       # evaluation score as reward because different answers hold different value for the user.
       state[:reward] = responsedict[:score]
     end
-    println("\nEvaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+
+    println("\nSQLLLM evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
     pprintln(Dict(responsedict))
 
+    # # store for later training
+    # responsedict[:thoughthistory] = state[:thoughtHistory]
+    # responsedict[:system] = systemmsg
+    # responsedict[:usermsg] = usermsg
+    # responsedict[:prompt] = prompt
+    # responsedict[:context] = context
+    # responsedict[:think] = think
+
+    # # read sessionId
+    # sessionid = JSON3.read("/appfolder/app/sessionid.json")
+    # # save to filename ./log/decisionlog.txt
+    # println("saving SQLLLM evaluator() to disk")
+    # filename = "agent_evaluator_log_$(sessionid[:id]).json"
+    # filepath = "/appfolder/app/log/$filename"
+    # # check whether there is a file path exists before writing to it
+    # if !isfile(filepath)
+    #   decisionlist = [responsedict]
+    #   println("Creating file $filepath")
+    #   open(filepath, "w") do io
+    #     JSON3.pretty(io, decisionlist)
+    #   end
+    # else
+    #   # read the file and append new data
+    #   decisionlist = copy(JSON3.read(filepath))
+    #   push!(decisionlist, responsedict)
+    #   println("Appending new data to file $filepath")
+    #   open(filepath, "w") do io
+    #     JSON3.pretty(io, decisionlist)
+    #   end
+    # end
+
     return responsedict[:score]
   end
   error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
@@ -705,15 +973,17 @@ function transition(state::T, args::NamedTuple
 
   decisionMakerF::Function = args[:decisionMaker]
   evaluatorF::Function = args[:evaluator]
-  reflector::Function = args[:reflector]
+  # reflector::Function = args[:reflector]
   context = args[:context]
   executeSQL::Function = args[:executeSQL]
   text2textInstructLLM::Function = args[:text2textInstructLLM]
-  insertSQLVectorDB::Function = args[:insertSQLVectorDB]
+  # insertSQLVectorDB::Function = args[:insertSQLVectorDB]
   querySQLVectorDBF::Function = args[:querySQLVectorDB]
+  llmFormatName::String = args[:llmFormatName]
 
   # getting SQL from vectorDB
-  thoughtDict = decisionMakerF(state, context, text2textInstructLLM; querySQLVectorDBF)
+  thoughtDict = decisionMakerF(state, context, text2textInstructLLM, llmFormatName; 
+                  querySQLVectorDBF)
 
   # map action and input() to llm function
   response =
@@ -727,7 +997,8 @@ function transition(state::T, args::NamedTuple
     elseif thoughtDict[:action_name] == "RUNSQL"
       response = SQLexecution(executeSQL, thoughtDict[:action_input])
       if response[:success]
-        extracted = extractContent_dataframe(response[:result], text2textInstructLLM, thoughtDict[:action_input])
+        extracted = extractContent_dataframe(response[:result], text2textInstructLLM, 
+          thoughtDict[:action_input], llmFormatName)
         (rawresponse=response[:result], result=extracted, errormsg=nothing, success=true)
       else
         (result=nothing, errormsg=response[:errormsg], success=false)
@@ -743,7 +1014,7 @@ function transition(state::T, args::NamedTuple
   reward::Integer = haskey(response, :reward) ? response[:reward] : 0
   isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false
   newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON3.write(result), select, reward, isterminal)
-  progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM)
+  progressvalue::Integer = evaluatorF(newstate, thoughtDict, text2textInstructLLM, llmFormatName)
 
   return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue)
 end
@@ -835,6 +1106,7 @@ julia> println(result)
 function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
   insertSQLVectorDB::Union{Function, Nothing}=nothing,
   similarSQLVectorDB::Union{Function, Nothing}=nothing,
+  llmFormatName="qwen3"
   )::NamedTuple{(:text, :rawresponse), Tuple{Any, Any}} where {T<:AbstractString}
 
   # use similarSQLVectorDB to find similar SQL for the query
@@ -844,7 +1116,8 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
     response = SQLexecution(executeSQL, sql)
     if response[:success]
       # intention = Dict(:intention=> "$(thoughtDict[:plan])")
-      extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql)
+      extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql, 
+                                          llmFormatName)
       return (text=extracted, rawresponse=response[:result]) 
     end
   end
@@ -873,7 +1146,6 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
   context = Dict(
     :tablelist =>
      """
-     Here are SQL that used to create tables in the database:
      create table customer (
       customer_id uuid primary key default gen_random_uuid (),
       customer_firstname varchar(128),
@@ -936,7 +1208,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
       wine_name varchar(128) not null,
       winery varchar(128) not null,
       vintage integer not null,
-      region varchar(128) not null,
+      region varchar(128) not null, -- A field used to store the name of a wine-producing region, such as Napa Valley (California), Bordeaux, Champagne, Tuscany, etc.
       country	varchar(128) not null,
       wine_type	varchar(128) not null,
       grape	varchar(128) not null,
@@ -946,7 +1218,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
       tannin integer,
       acidity integer,
       fizziness	integer,
-      tasting_notes text,
+      tasting_notes text, -- A field used to record the distinctive flavors of wine such as floral, citrus, apple, earthy, daisy, etc. 
       note text,
       other_attributes jsonb,
       
@@ -997,15 +1269,16 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
                     text2textInstructLLM=text2textInstructLLM,
                     querySQLVectorDB=similarSQLVectorDB,
                     insertSQLVectorDB=insertSQLVectorDB,
+                    llmFormatName=llmFormatName
                   )
 
   earlystop(state) = state[:reward] >= 8 ? true : false
 
   root, _, resultState, highValueState = 
     LLMMCTS.runMCTS(initialstate, transition, transitionargs;
-                  horizontalSampleExpansionPhase=3, 
-                  horizontalSampleSimulationPhase=3,
-                  maxSimulationDepth=5,
+                  horizontalSampleExpansionPhase=1, 
+                  horizontalSampleSimulationPhase=1,
+                  maxSimulationDepth=1,
                   maxiterations=1, 
                   explorationweight=1.0,
                   earlystop=earlystop,
@@ -1013,17 +1286,17 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
                   multithread=false)
 
   # compare all high value state answer then select the best one
-  if length(highValueState) > 0
+  if length(highValueState) > 1
     # open("/appfolder/app/highValueState.json", "w") do io
     #   JSON3.pretty(io, highValueState)
     # end
-    selected = compareState(query, highValueState, text2textInstructLLM)
-    resultState = highValueState[selected]  #BUG compareState() select 0
+    selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
+    resultState = highValueState[selected]
   end
   latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation")
   action_input = Symbol("action_input_$latestInd")  # latest sql
   sql = resultState[:thoughtHistory][action_input]
-  extracted = resultState[:thoughtHistory][latestKey]
+  extractedTableContent = resultState[:thoughtHistory][latestKey]
 
   # add to vectorDB only if the answer is achieved and the state is terminal 
   if insertSQLVectorDB !== nothing && resultState[:isterminal] == true &&
@@ -1032,11 +1305,11 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
     insertSQLVectorDB(resultState[:thoughtHistory][:question], sql)
   end
 
-  if extracted === nothing
-    println("query() return nothing")
+  if extractedTableContent === nothing
+    println("\nSQLLLM query() return nothing ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
   end
 
-  result = (text=extracted, rawresponse=resultState[:rawresponse])
+  result = (text=extractedTableContent, rawresponse=resultState[:rawresponse])
 
   return result 
 end
@@ -1059,7 +1332,7 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
   reward::T3, isterminal::Bool
   )::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
   
-  keys = [:comprehension, :action_name, :action_input, :observation]
+  keys = [:plan, :action_name, :action_input, :observation]
   # latestKeys = []
 
   currentstate_latestKey, currentstate_latestIndice = 
@@ -1090,8 +1363,9 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
 end
 
 
-function generatequestion(state::T1, context, text2textInstructLLM::Function;
-  similarSQL::Union{T2, Nothing}=nothing, maxattempt=10
+function generatequestion(state::T1, context, text2textInstructLLM::Function, 
+  llmFormatName::String;
+  similarSQL::Union{T2, Nothing}=nothing, maxattempt=10,
   )::String where {T1<:AbstractDict, T2<:AbstractString}
 
   similarSQL = 
@@ -1122,37 +1396,37 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
     4) Do not generate any question or comments at the end.
 
     You should follow the following guidelines:
-    - When querying data in the database, start with broad search terms and refine your query later for more precise results. 
+    - If there is no search result from the database, remove the restrictive criteria until a search result is available, and proceed from there.
 
     You should then respond to the user with:
-    1) Understanding: 
-        - State your understanding about the current situation.
-    2) Q: Given the situation, "ask yourself" about the situation at least five, but no more than ten, questions.
-    3) A: Given the situation, "answer to yourself" the best you can.
+    1) Q: Given the situation, "ask yourself" about the situation at least three, but no more than five, questions.
+    2) A: Given the situation, "answer to yourself" the best you can.
         - Do not generate any text after the last answer.
     
     You must only respond in format as described below:
-    Understanding: ...
     Q1: ...
     A1: ...
     Q2: ...
     A2: ...
-    Q3: ...
-    A3: ...
     ...
 
     Here are some examples:
     Q: What information in the hints is not necessary based on the query?
     A: Country is not specified in the query thus it should not be included in an SQL
-
     Q: How can I modify a SQL example to fit my specific query needs?
     A: ...
+    Q: Why the query failed?
+    A: ...
+    Q: What criteria become more restrictive as the search scope broadens and can be remove?
+    A: In the "2019 Toyota Camry hybrid" search query, "2019" represents the most restrictive criteria because it narrows the data scope to a specific year, whereas "Toyota" and "Camry" are broader categories that allow for more general results.
+    Q: What works and what not previously?
+    A: ...
 
     Let's begin!
   """
 
-  header = ["Understanding:", "Q1:"]
-  dictkey = ["understanding", "q1"]
+  header = ["Q1:"]
+  dictkey = ["q1"]
   
   workprogress = ""
   for (k, v) in state[:thoughtHistory]
@@ -1162,7 +1436,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
   end
 
   response = nothing # store for show when error msg show up
-  errornote = ""
+  errornote = "N/A"
 
   for attempt in 1:maxattempt
     usermsg =
@@ -1172,6 +1446,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
     Example: $similarSQL
     Your work progress: $workprogress
     P.S. $errornote
+    /no_think
     """
 
     _prompt = 
@@ -1181,10 +1456,11 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
     ]
 
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
 
     response = text2textInstructLLM(prompt, modelsize="medium")
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
     
     # check if response is valid
     q_number = count("Q", response)

src/llmfunction.jl

@@ -347,8 +347,9 @@ end
 
 # Signature
 """
-function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function
-)::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}}
+function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function,
+  llmFormatName::String
+  )::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}}
 
   Hints = "None"
 
@@ -366,17 +367,14 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
   - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search.
 
   You should then respond to the user with:
-  1) Comprehension: 
-      - State your comprehension about the current situation.
-  3) Plan: Step-by-step instructions of how to complete the task.
+  1) Plan: Step-by-step instructions of how to complete the task.
       - Focus on improving the code from the last round.
       - Do not create any table in the database.
-  4) Code:
+  2) Code:
       - Write new improved code.
       - Do not wrap the code and no comment as it will be executed directly without any modification against the database.
 
   You should only respond in format as described below and nothing more:
-  Comprehension: ...
   Plan: 
     1) ...
     2) ... 
@@ -406,13 +404,14 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
       ]
 
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
     try
       response = text2textInstructLLM(prompt, modelsize="medium")
-      response = GeneralUtils.deFormatLLMtext(response, "granite3")
-
-      header = ["Comprehension:", "Plan:", "Code:"]
-      dictkey = ["comprehension", "plan", "code"]
+      response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+      think, response = GeneralUtils.extractthink(response)
+      
+      header = ["Plan:", "Code:"]
+      dictkey = ["plan", "code"]
 
       responsedict = GeneralUtils.textToDict(response, header; 
                                             dictKey=dictkey, symbolkey=true)
@@ -518,9 +517,9 @@ function SQLexecution(executeSQL::Function, sql::T
     tablesize = size(df)
     row, column = tablesize
     if row == 0
-      error("The resulting table has 0 row. Possible causes: 1) Your search criteria might be too specific. Relaxing some conditions could yield better results. Remember, you can always refine your search later. 2) There could be a typo in your search query. 3) You might be searching in the wrong place.")
+      error("\nThe resulting table has 0 row. Please try again.")
     elseif column > 30
-      error("SQL execution failed. An unexpected error occurred. Please try again.")
+      error("\nSQL execution failed. An unexpected error occurred. Please try again.")
     end
 
     df1 =
@@ -561,8 +560,9 @@ end
 
 # Signature
 """
-function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String
-)::String
+function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String,
+  llmFormatName::String
+  )::String
   tablesize = size(df)
   row = tablesize[1]
   column = tablesize[2]
@@ -628,23 +628,26 @@ function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function,
     ]
 
   # put in model format
-  prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+  prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
   header = ["About_resulting_table:", "Search_summary:"]
   dictkey = ["about_resulting_table", "search_summary"]
 
   for i in 1:5
     response = text2textInstructLLM(prompt, modelsize="medium")
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
 
-    kw = []
-    # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
-    for keyword in header
-      detected = GeneralUtils.detect_keyword(keyword, response)
-      push!(kw, detected)
-    end
-    if nothing ∈ kw
-      println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue  # try again next loop
+    # check whether response has all header
+    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
+    missingkeys = [k for (k, v) in detected_kw if v === nothing]
+    if !isempty(missingkeys)
+      errornote = "$missingkeys are missing from your previous response"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
+      errornote = "\nYour previous attempt has duplicated points according to the required response format"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
     end
 
     responsedict = GeneralUtils.textToDict(response, header; 
@@ -736,7 +739,9 @@ julia> result = SQLLLM.getTableNameFromSQL(sql, text2textInstructLLM)
 
 # Signature
 """
-function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{String} where {T<:AbstractString}
+function getTableNameFromSQL(sql::T, text2textInstructLLM::Function,
+  llmFormatName::String
+  )::Vector{String} where {T<:AbstractString}
   systemmsg = """
               Extract table name out of the user query.
 
@@ -764,14 +769,14 @@ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{Str
     ]
 
   # put in model format
-  prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+  prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
   header = ["Table_name:"]
   dictkey = ["table_name"]
 
   for attempt in 1:5
     try
       response = text2textInstructLLM(prompt, modelsize="medium")
-      response = GeneralUtils.deFormatLLMtext(response, "granite3")
+      response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
       responsedict = GeneralUtils.textToDict(response, header; 
                                             dictKey=dictkey, symbolkey=true)
       response = copy(JSON3.read(responsedict[:table_name]))
@@ -820,44 +825,39 @@ julia>
 - The LLM evaluates attempts based on accuracy and relevance to the original question
 """
 function compareState(question::String, highValueStateList::Vector{T},
-                      text2textInstructLLM::Function)::Integer where {T<:AbstractDict}
+  text2textInstructLLM::Function, llmFormatName::String
+  )::Integer where {T<:AbstractDict}
 
   systemmsg = 
   """
-    <Your profile>
+    Your profile:
     - You are a helpful assistant
-    </Your profile>
-    <Situation>
-    The user has made multiple attempts to solve the question, resulting in various answers
-    <Your mission>
+    Situation:
+    - The user has made multiple attempts to solve the question, resulting in various answers
+    Your mission:
     - Identify and select the most accurate and relevant response from these multiple results for the user
-    </Your mission>
-    <At each round of conversation, you will be given the following>
+    At each round of conversation, you will be given the following:
       Question: the question the user is trying to answer
       Attempt: the user's attempted actions and their corresponding results
-    </At each round of conversation, you will be given the following>
-    <You should then respond to the user with the following>
-      Comparison: a comparison of all results from all attempts
+    You should then respond to the user with the following:
+      Comparison: detailed comparison of all results from all attempts from various aspects.
       Rationale: a brief explanation of why the selected response is the most accurate and relevant
       Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
-    </You should then respond to the user with the following>
-    <You should only respond in format as described below>
+    You should only respond in format as described below:
       Comparison: ...
       Rationale: ...
       Selected_response_number: ...
-    </You should only respond in format as described below>
-    <Here are some examples>
-    User's question: "How many German wines do you have?"
-    Attempt 1:
-    Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
-    Result: 100 wines
-    Attempt 2:
-    Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
-    Result: 50 red wines
-    Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
-    Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
-    Selected_response_number:1
-    </Here are some examples>
+    Here are some examples:
+      User's question: "How many German wines do you have?"
+      Attempt 1)
+      Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
+      Result: 100 wines
+      Attempt 2)
+      Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
+      Result: 50 red wines
+      Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
+      Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
+      Selected_response_number:1
 
     Let's begin!
   """
@@ -880,26 +880,26 @@ function compareState(question::String, highValueStateList::Vector{T},
 
   """
   # put potential solutions from potentialSolution into the following form
-  Attempt 1
-  action_name: 
-  action_input:
-  observation:
-  Attempt 2
-  action_name: 
-  action_input:
-  observation:
+  Attempt 1)
+    action_name: 
+    action_input:
+    observation:
+  Attempt 2)
+    action_name:` 
+    action_input:
+    observation:`
   ...
   """
   potentialSolutionStr = ""
   for (i, state) in enumerate(potentialSolution)
-    potentialSolutionStr *= "Attempt $i\n"
+    potentialSolutionStr *= "Attempt $i)\n"
     for k in keys
       potentialSolutionStr *= "$k: $(state[k])\n"
       println("")
     end
   end
 
-  errornote = ""
+  errornote = "N/A"
 
   for attempt in 1:10
     errorFlag = false
@@ -918,7 +918,7 @@ function compareState(question::String, highValueStateList::Vector{T},
     ]
 
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
 
     header = ["Comparison:", "Rationale:", "Selected_response_number:"]
     dictkey = ["comparison", "rationale", "selected_response_number"]
@@ -928,19 +928,20 @@ function compareState(question::String, highValueStateList::Vector{T},
     # sometime LLM output something like **Comprehension**: which is not expected
     response = replace(response, "**"=>"")
     response = replace(response, "***"=>"")
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
 
-    # make sure every header is in the response
-    for i in header
-      detected = GeneralUtils.detect_keyword(i, response)
-      if detected === nothing
-        errornote = "Your previous attempt didn't provide $i"
-        errorFlag = true
-      end
-    end
-    if errorFlag  
-      println("\nERROR SQLLLM compareState() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue  # skip to the next iteration
+    # check whether response has all header
+    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
+    missingkeys = [k for (k, v) in detected_kw if v === nothing]
+    if !isempty(missingkeys)
+      errornote = "$missingkeys are missing from your previous response"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
+      errornote = "\nYour previous attempt has duplicated points according to the required response format"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
     end
 
     responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=true)

system_prompt_template.jl

@@ -1,53 +1,60 @@
 """
-Default system message template:
+# -------------------------------- Default system message template ------------------------------- #
 
-  <Your role>
-    - You are a helpful assistant
-  </Your role>
-  <Situation>
-    - Describe the current situation
-  </Situation>
-  <Your vision>
-    - state your vision of how the situation will evolve, what would you want the situation to evolve into
-  </Your vision>
-  <Your mission>
-    - state the goal
-  </Your mission>
-  <Your mission's objective includes>
-    - Break the goal into smaller steps
-  </Your mission's objective includes>
-  <Your responsibility includes>
-    - state the mini goals that fall under your responsibility
-  </Your responsibility includes>
-  <Your responsibility does NOT includes>
+<Your role>
+  - You are a helpful assistant
+</Your role>
+<Situation>
+  - Describe the current situation
+  Ex. The world use enormous energy from non-sustainable sources. This leads to climate change.
+</Situation>
+<Your vision>
+  - state your vision of how the situation will evolve, what would you want the situation to evolve into
+  Ex. To be the leading innovator in sustainable technology by 2030, transforming global energy systems.
+</Your vision>
+<Your mission>
+  - state the goal
+  Ex. Empowering communities through clean energy solutions to create a sustainable future.
+</Your mission>
+<Your mission's objective includes>
+  - Specific, measurable, and time-bound goals that directly support the mission.
+  Ex. Launch 50 solar-powered water purification systems in 3 regions by 2025.
+</Your mission's objective includes>
+<Your responsibility includes>
+  - state the mini goals that fall under your responsibility
+</Your responsibility includes>
+<Your responsibility does NOT includes>
   - 
-  </Your responsibility does NOT includes>
-  <At each round of conversation, you will be given the following information>
+</Your responsibility does NOT includes>
+<At each round of conversation, you will be given the following information>
+  -
+</At each round of conversation, you will be given the following information>
+<You must follow the following policy>
+  -
+</You must follow the following policy>
+<You should follow the following guidelines>
+  -
+</You should follow the following guidelines>
+<You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+  Comprehension: State your comprehension about the current situation.
+  Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+  Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names:
+    - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
+    - CHECKRESOURCES which you can use to check resources
+    - IMPLEMENT which you can use to implement the solution
+  Action_input: Detail the input for the action.
+</You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+<You should only respond in format as described below>
+  Comprehension: ...
+  Plan: ...
+  Action_name: ...
+  Action_input: ...
+</You should only respond in format as described below>
+<Here are some examples>
 
-  </At each round of conversation, you will be given the following information>
-  <You must follow the following guidelines>
-    -
-  </You must follow the following guidelines>
-  <You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
-    Comprehension: State your comprehension about the current situation.
-    Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
-    Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names:
-      - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
-      - CHECKRESOURCES which you can use to check resources
-      - IMPLEMENT which you can use to implement the solution
-    Action_input: Detail the input for the action.
-  </You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
-  <You should only respond in format as described below>
-    Comprehension: ...
-    Plan: ...
-    Action_name: ...
-    Action_input: ...
-  </You should only respond in format as described below>
-  <Here are some examples>
+</Here are some examples>
 
-  </Here are some examples>
-
-  Let's begin!
+Let's begin!
 
 
 
@@ -57,7 +64,7 @@ Default system message template:
 
 
 
-Example:
+# ------------------------------------------- Example: ------------------------------------------- #
 
 <Your profile>
 - You are a founder of a tech startup

test/runtests.jl

@@ -3,7 +3,7 @@ using LibPQ, Dates, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Ba
 using GeneralUtils, SQLLLM
 
 
-config = copy(JSON3.read("/appfolder/mountvolume/appdata/config.json"))
+config = JSON3.read("/appfolder/app/dev/YiemAgent/test/config.json")
 
 function executeSQL(sql::T) where {T<:AbstractString}
   host = config[:externalservice][:wineDB][:host]
@@ -29,13 +29,19 @@ function executeSQLVectorDB(sql)
   return result
 end
 
-function text2textInstructLLM(prompt::String; maxattempt=3)
+function text2textInstructLLM(prompt::String; maxattempt::Integer=3, modelsize::String="medium",
+    senderId=GeneralUtils.uuid4snakecase(), timeout=180,
+    llmkwargs=Dict(
+      :num_ctx => 32768,
+      :temperature => 0.5,
+    )
+  )
   msgMeta = GeneralUtils.generate_msgMeta(
     config[:externalservice][:loadbalancer][:mqtttopic];
     msgPurpose="inference",
     senderName="yiemagent",
-    senderId=sessionId,
-    receiverName="text2textinstruct_small",
+    senderId=senderId,
+    receiverName="text2textinstruct_$modelsize",
     mqttBrokerAddress=config[:mqttServerInfo][:broker],
     mqttBrokerPort=config[:mqttServerInfo][:port],
   )
@@ -44,16 +50,13 @@ function text2textInstructLLM(prompt::String; maxattempt=3)
     :msgMeta => msgMeta,
     :payload => Dict(
       :text => prompt,
-      :kwargs => Dict(
-        :num_ctx => 16384,
-        :temperature => 0.2,
-      )
+      :kwargs => llmkwargs
     )
   )
 
   response = nothing
   for attempts in 1:maxattempt
-    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=300, maxattempt=2)
+    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=timeout, maxattempt=maxattempt)
     payload = _response[:response]
     if _response[:success] && payload[:text] !== nothing
       response = _response[:response][:text]
@@ -76,7 +79,7 @@ function getEmbedding(text::T) where {T<:AbstractString}
     msgPurpose="embedding",
     senderName="yiemagent",
     senderId=sessionId,
-    receiverName="text2textinstruct_small",
+    receiverName="textembedding",
     mqttBrokerAddress=config[:mqttServerInfo][:broker],
     mqttBrokerPort=config[:mqttServerInfo][:port],
   )
@@ -87,7 +90,8 @@ function getEmbedding(text::T) where {T<:AbstractString}
       :text => [text] # must be a vector of string
     )
   )
-  response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
+
+  response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120, maxattempt=3)
   embedding = response[:response][:embeddings]
   return embedding
 end
@@ -108,7 +112,6 @@ function findSimilarTextFromVectorDB(text::T1, tablename::T2, embeddingColumnNam
   return df
 end
 
-
 function similarSQLVectorDB(query; maxdistance::Integer=100)
   tablename = "sqlllm_decision_repository"
   # get embedding of the query
@@ -131,7 +134,6 @@ function similarSQLVectorDB(query; maxdistance::Integer=100)
   end
 end
 
-
 function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString}
   tablename = "sqlllm_decision_repository"
   # get embedding of the query
@@ -155,7 +157,65 @@ function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1
   end
 end
 
-sessionId = "555"
+
+function similarSommelierDecision(recentevents::T1; maxdistance::Integer=3
+  )::Union{AbstractDict, Nothing} where {T1<:AbstractString}
+  tablename = "sommelier_decision_repository"
+  # find similar
+  println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__)
+  df = findSimilarTextFromVectorDB(recentevents, tablename, 
+                                  "function_input_embedding", executeSQLVectorDB)
+  row, col = size(df)
+  distance = row == 0 ? Inf : df[1, :distance]
+  if row != 0 && distance < maxdistance
+    # if there is usable decision, return it.
+    rowid = df[1, :id]
+    println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
+    output_b64 = df[1, :function_output_base64]   # pick the closest match
+    _output_str = String(base64decode(output_b64))
+    output = copy(JSON3.read(_output_str))
+    return output
+  else
+    println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
+    return nothing
+  end
+end
+
+
+function insertSommelierDecision(recentevents::T1, decision::T2; maxdistance::Integer=5
+  ) where {T1<:AbstractString, T2<:AbstractDict}
+  tablename = "sommelier_decision_repository"
+  # find similar
+  df = findSimilarTextFromVectorDB(recentevents, tablename, 
+                                  "function_input_embedding", executeSQLVectorDB)
+  row, col = size(df)
+  distance = row == 0 ? Inf : df[1, :distance]
+  if row == 0 || distance > maxdistance # no close enough SQL stored in the database
+    recentevents_embedding = getEmbedding(recentevents)[1]
+    recentevents = replace(recentevents, "'" => "")
+    decision_json = JSON3.write(decision)
+    decision_base64 = base64encode(decision_json)
+    decision = replace(decision_json, "'" => "")
+    
+    sql = """
+          INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding');
+          """
+    println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__)
+    println(sql)
+    _ = executeSQLVectorDB(sql)
+  else
+    println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__)
+  end
+end
+
+sessionId = GeneralUtils.uuid4snakecase()
+d = Dict(:id => sessionId)
+filepath = "/appfolder/app/sessionid.json"
+open(filepath, "w") do io
+  JSON3.pretty(io, d)
+end
+
+
 
 
 # query = "How many German wines do you have?"
@@ -165,8 +225,9 @@ sessionId = "555"
 
 
 # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?")
-query = "How many French wines from Yiem store under 100 dollars do you have?"
+# query = "How many French wines from Yiem store under 100 dollars do you have?"
 # query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40"
+query = "from Yiem retailer, red wine from France. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5"
 # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza"
 # query = "wine_type: white, country: Austria, food to be served with wine: pork"
 # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"