update

2025-03-18 21:22:12 +07:00
parent 7fd0d6269a
commit e6ce6f9954
7 changed files with 375 additions and 110 deletions
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised

-julia_version = "1.11.3"
+julia_version = "1.11.4"
 manifest_format = "2.0"
 project_hash = "9e0d7dca51b949f2ffa5477b895b90988ec62529"

@@ -202,7 +202,7 @@ version = "1.11.0"
 deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON3", "MQTTClient", "PrettyPrinting", "Random", "SHA", "UUIDs"]
 path = "../GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.2.2"
+version = "0.2.3"

 [[deps.HTTP]]
 deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
@@ -306,7 +306,7 @@ version = "1.19.3+0"
 deps = ["GeneralUtils", "JSON3", "PrettyPrinting"]
 path = "../LLMMCTS"
 uuid = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
-version = "0.1.3"
+version = "0.1.4"

 [[deps.LaTeXStrings]]
 git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"
@@ -471,7 +471,7 @@ version = "0.3.27+1"
 [[deps.OpenLibm_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
-version = "0.8.1+2"
+version = "0.8.1+4"

 [[deps.OpenSSL]]
 deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"]
--- a/src/interface.jl
+++ b/src/interface.jl
@@ -141,7 +141,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
    For your information:
    - Observation: Result of the immediately preceding action

-    At each round of conversation, the user will give you the current situation:
+    At each round of conversation, the user will give you the following:
    User Query: ...
    Example: ...
    Your Q&A: ...
@@ -164,7 +164,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
      - State your comprehension about the current situation.
    2) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
    3) Action_name (Must be aligned with your plan): Can be one of the following functions: 
-        - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database.
+        - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
          For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
          Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
    4) Action_input: Input to the action
@@ -300,7 +300,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
      responsedict[:action_input] = sql
    end

-    toollist = ["TABLEINFO", "GETDATA"]
+    toollist = ["TABLEINFO", "RUNSQL"]
    if responsedict[:action_name] ∉ toollist
      errornote = "\nYou must only use the given functions"
      println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
@@ -340,6 +340,244 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
  end
  error("DecisionMaker failed to generate a thought \n", response)
 end
+# function decisionMaker(state::T1, context, text2textInstructLLM::Function,
+#   ; querySQLVectorDBF::Union{T2, Nothing}=nothing
+#   )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:Function}
+
+#   # lessonDict =
+#   # if isfile("lesson.json")
+#   #   lessonDict = copy(JSON3.read("lesson.json"))
+#   # else
+#   #   lessonDict = nothing
+#   # end
+
+#   # lessonDict = nothing
+
+#   # lesson =
+#   # if lessonDict === nothing
+#   #   ""
+#   # else
+#   #   """
+#   #   You have attempted to help the user before and failed, either because your reasoning for the 
+#   #   recommendation was incorrect or your response did not exactly match the user expectation. 
+#   #   The following lesson(s) give a plan to avoid failing to help the user in the same way you 
+#   #   did previously. Use them to improve your strategy to help the user.
+
+#   #   Here are some lessons in JSON format:
+#   #   $(JSON3.write(lessonDict))
+
+#   #   When providing the thought and action for the current trial, that into account these failed 
+#   #   trajectories and make sure not to repeat the same mistakes and incorrect answers. 
+#   #   """
+#   # end
+
+#   systemmsg = 
+#   """
+#     You are a helpful assistant that find the data from a database to satisfy the user's query. 
+#     You are also eager to improve your helpfulness.
+
+#     For your information:
+#     - Observation: Result of the immediately preceding action
+
+#     At each round of conversation, the user will give you the current situation:
+#     User Query: ...
+#     Example: ...
+#     Your Q&A: ...
+#     Your work progress: ...
+#     Evaluation: Evaluation of the immediately preceding action and observation
+#     Suggestion: Suggestion for the immediately preceding action and observation
+    
+#     You must follow the following guidelines:
+#     - Keep SQL queries focused only on the provided information.
+
+#     You should follow the following guidelines:
+#     - Do not create any table in the database
+#     - A junction table can be used to link tables together. Another use case is for filtering data.
+#     - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
+#     - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
+#     - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
+
+#     You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action:
+#     1) Comprehension: 
+#       - State your comprehension about the current situation.
+#     2) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+#     3) Action_name (Must be aligned with your plan): Can be one of the following functions: 
+#         - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database.
+#           For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
+#           Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
+#     4) Action_input: Input to the action
+
+#     You should only respond in format as described below:
+#     Comprehension: ...
+#     Plan: ...
+#     Action_name: ...
+#     Action_input: ...
+
+#     Let's begin!
+#   """
+  
+#   workprogress = ""
+#   for (k, v) in state[:thoughtHistory]
+#     if k ∉ [:question]
+#       workprogress *= "$k: $v\n"
+#     end
+#   end
+  
+#   response = nothing # store for show when error msg show up
+#   errornote = ""
+
+#   # provide similar sql only for the first attempt
+#   similarSQL_ = "None"
+#   if length(state[:thoughtHistory]) == 1
+#     sql, distance = querySQLVectorDBF(state[:thoughtHistory][:question])
+#     similarSQL_ = sql !== nothing ? sql : "None"
+#   end
+
+
+#   for attempt in 1:10
+#     QandA = generatequestion(state, context, text2textInstructLLM; similarSQL=similarSQL_)
+
+#     usermsg =
+#     """
+#     $(context[:tablelist])
+#     User query: $(state[:thoughtHistory][:question])
+#     Example: $similarSQL_
+#     Your Q&A: $QandA
+#     Your work progress: $workprogress
+#     Evaluation: $(state[:evaluation])
+#     Suggestion: $(state[:suggestion])
+#     $errornote
+#     """
+
+#     _prompt = 
+#     [
+#       Dict(:name=> "system", :text=> systemmsg),
+#       Dict(:name=> "user", :text=> usermsg)
+#     ]
+
+#     # put in model format
+#     prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
+#     response = text2textInstructLLM(prompt)
+
+#     # LLM tends to generate observation given that it is in the input
+#     response = 
+#       if occursin("observation:", response)
+#         string(split(response, "observation:")[1])
+#       elseif occursin("Observation:", response)
+#         string(split(response, "Observation:")[1])
+#       elseif occursin("observation_", response)
+#         string(split(response, "observation_")[1])
+#       elseif occursin("Observation_", response)
+#         string(split(response, "Observation_")[1])
+#       else
+#         response
+#       end
+
+#     # sometime LLM output something like **Comprehension**: which is not expected
+#     response = replace(response, "**"=>"")
+#     response = replace(response, "***"=>"")
+
+#     # some time LLM output Plan_1: so we need to detect and replace topic numbering
+#     regex = r"_[0-1000]+:"
+#     matches = collect(eachmatch(regex, response))
+#     for m in matches
+#       response = replace(response, string(m.match)=>":") 
+#     end
+
+#     if occursin("NULL", response)
+#       errornote = "\nSQL decisionMaker() NULL response is not allowed"
+#       println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue
+#     end
+
+#     header = ["Comprehension:", "Plan:", "Action_name:", "Action_input:"]
+#     dictkey = ["comprehension", "plan", "action_name", "action_input"]
+
+#     # detect if there are more than 1 key per categories
+#     wordcount = GeneralUtils.countGivenWords(response, header)
+#     duplicateKeywordFlag = false
+#     for (i, v) in enumerate(wordcount)
+#       keyword = header[i]
+#       keywordNumber = v
+#       if keywordNumber > 1
+#         errornote = "\nSQL query has duplicated keyword, $keyword"
+#         println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#         duplicateKeywordFlag = true
+#         break
+#       end
+#     end
+#     duplicateKeywordFlag == true ? continue : nothing
+
+#     # check whether response has all header
+#     kw = []
+#     # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
+#     for keyword in header
+#       detected = GeneralUtils.detect_keyword(keyword, response)
+#       push!(kw, detected)
+#     end
+#     if nothing ∈ kw
+#       println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue  # try again next loop
+#     end
+
+#     # textToDict() search for action_input
+#     responsedict = GeneralUtils.textToDict(response, header; 
+#                                             dictKey=dictkey, symbolkey=true)
+
+#     delete!(responsedict, :observation)
+
+#     # remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String})
+#     if occursin("```", responsedict[:action_input])
+#       sql = GeneralUtils.extract_triple_backtick_text(responsedict[:action_input])[1]
+#       if sql[1:4] == "sql\n"
+#         sql = sql[5:end]
+#       end
+#       sql = split(sql, ';')   # some time there are comments in the sql
+#       sql = sql[1] * ';'
+
+#       responsedict[:action_input] = sql
+#     end
+
+#     toollist = ["TABLEINFO", "GETDATA"]
+#     if responsedict[:action_name] ∉ toollist
+#       errornote = "\nYou must only use the given functions"
+#       println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#       continue
+#     end
+
+#     for i in toollist
+#       if occursin(i, responsedict[:action_input])
+#         errornote = "\n action_name is in action_input which is not allowed."
+#         println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#         continue
+#       end
+#     end
+
+#     for i ∈ [:comprehension, :plan, :action_name, :action_input]
+#       if length(JSON3.write(responsedict[i])) == 0
+#         errornote = "\n $i is empty"
+#         println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#         continue
+#       end
+#     end
+
+#     # check if there are more than 1 key per categories
+#     for i ∈ [:comprehension, :plan, :action_name, :action_input]
+#       matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
+#       if length(matchkeys) > 1
+#         errornote = "\n $i has more than one key"
+#         println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+#         continue
+#       end
+#     end
+
+#     state[:decisionMaker] = responsedict
+
+#     return responsedict
+
+#   end
+#   error("DecisionMaker failed to generate a thought \n", response)
+# end

 """ Assigns a scalar value to each new child node to be used for selec-
 tion and backpropagation. This value effectively quantifies the agent's progress in task completion,
@@ -374,9 +612,9 @@ function evaluator(state::T1, text2textInstructLLM::Function
  "reasoning" is agent's step-by-step reasoning about the current situation
  "plan" is agent's plan to complete the task from the current situation
  "action_name" is the name of the action taken, which can be one of the following functions:
-      - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database.
-          For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
-          Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
+      - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
+        For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
+        Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
  "action_input" is the input to the action
  "observation" is result of the preceding immediate action

@@ -722,7 +960,7 @@ function transition(state::T, args::NamedTuple
    elseif thoughtDict[:action_name] == "TABLEINFO"
      input = thoughtDict[:action_input]
      tableinfo(executeSQL, input)
-    elseif thoughtDict[:action_name] == "GETDATA"
+    elseif thoughtDict[:action_name] == "RUNSQL"
      response = SQLexecution(executeSQL, thoughtDict[:action_input])
      if response[:success]
        extracted = extractContent_dataframe(response[:result], text2textInstructLLM, thoughtDict[:action_input])
@@ -877,19 +1115,20 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
    LLMMCTS.runMCTS(initialstate, transition, transitionargs;
                  horizontalSampleExpansionPhase=5, 
                  horizontalSampleSimulationPhase=2,
-                  maxSimulationDepth=5, 
+                  maxSimulationDepth=10, 
                  maxiterations=1, 
                  explorationweight=1.0,
                  earlystop=earlystop,
                  saveSimulatedNode=true,
                  multithread=true)

-  #[WORKING] compare all high value state answer then select the best one
+  # compare all high value state answer then select the best one
  if length(highValueState) > 0
-    open("/appfolder/app/highValueState.json", "w") do io
-      JSON3.pretty(io, highValueState)
-    end
-    resultState = compareState(query, highValueState)
+    # open("/appfolder/app/highValueState.json", "w") do io
+    #   JSON3.pretty(io, highValueState)
+    # end
+    selected = compareState(query, highValueState, text2textInstructLLM)
+    resultState = highValueState[selected]
  end
  latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation")
  action_input = Symbol("action_input_$latestInd")  # latest sql
@@ -936,7 +1175,7 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
  nextindice = currentstate_latestKey !== nothing ? currentstate_latestIndice + 1 : 1
    # currentstate_latestKey == :NA ? 1 : currentstate_latestIndice + 1

-  currentstate_latestKey = makeNextKey.(keys, nextindice)
+  currentstate_latestKey = makekey.(keys, nextindice)

  # add Thought, action, observation to thoughtHistory
  newstate = deepcopy(currentstate)
@@ -959,9 +1198,6 @@ function makeNewState(currentstate::T1, thoughtDict::T4, rawresponse, response::
 end


-makeNextKey(key, indice) = Symbol("$(key)_$indice")
-
-
 function generatequestion(state::T1, context, text2textInstructLLM::Function;
  similarSQL::Union{T2, Nothing}=nothing
  )::String where {T1<:AbstractDict, T2<:AbstractString}
--- a/src/llmfunction.jl
+++ b/src/llmfunction.jl
@@ -4,7 +4,7 @@ export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox,
  getTableNameFromSQL, extractContent_dataframe, SQLexecution, compareState

 using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
-  DataStructures, StatsBase
+  DataStructures, StatsBase, Dates
 using GeneralUtils, LLMMCTS
 using ..util

@@ -812,8 +812,8 @@ julia>

 # Signature
 """
-function compareState(query, highValueStateList)
-
+function compareState(question::String, highValueStateList, text2textInstructLLM::Function)
+  println(typeof(highValueStateList))
  systemmsg = 
  """
    <Your profile>
@@ -825,14 +825,13 @@ function compareState(query, highValueStateList)
    - Identify and select the most accurate and relevant response from these multiple results for the user
    </Your mission>
    <At each round of conversation, you will be given the following>
-      - The user's question
-      - The user's attempted actions and their corresponding results
+      Question: the question the user is trying to answer
+      Attempt: the user's attempted actions and their corresponding results
    </At each round of conversation, you will be given the following>
    <You should then respond to the user with the following>
-      Comparison: a comparison of the results from each attempt
+      Comparison: a comparison of all results from all attempts
      Rationale: a brief explanation of why the selected response is the most accurate and relevant
-      Selected_response_number: the number the selected response in the list of results
-      
+      Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
    </You should then respond to the user with the following>
    <You should only respond in format as described below>
      Comparison: ...
@@ -855,89 +854,100 @@ function compareState(query, highValueStateList)
    Let's begin!
  """

-  # thoughthistory = ""
-  # for (k, v) in state[:thoughtHistory]
-  #   thoughthistory *= "$k: $v\n" 
-  # end
+  potentialSolution = []
+  keys = [:action_input, :observation]
+  # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
+  for state in highValueStateList
+    thoughtHistory = state[:thoughtHistory]
+    _, currentstate_latestIndice = 
+      GeneralUtils.findHighestIndexKey(thoughtHistory, keys[1])
+    latestKeys = makekey.(keys, currentstate_latestIndice)
+    d = Dict()
+    # get the last action_name, action_input, observation of currentstate
+    for (i,v) in enumerate(keys)
+      d[v] = thoughtHistory[latestKeys[i]]
+    end
+    push!(potentialSolution, d)
+  end

-  # errornote = ""
+  """
+  # put potential solutions from potentialSolution into the following form
+  Attempt 1
+  action_name: 
+  action_input:
+  observation:
+  Attempt 2
+  action_name: 
+  action_input:
+  observation:
+  ...
+  """
+  potentialSolutionStr = ""
+  for (i, state) in enumerate(potentialSolution)
+    potentialSolutionStr *= "Attempt $i\n"
+    for k in keys
+      potentialSolutionStr *= "$k: $(state[k])\n"
+      println("")
+    end
+  end

-  # for attempt in 1:10
-  #   errorFlag = false
+  errornote = ""

-  #   usermsg =
-  #   """
-  #   Trajectory: $thoughthistory
-  #   Error_note: $errornote
-  #   """
+  for attempt in 1:10
+    errorFlag = false

-  #   _prompt = 
-  #   [
-  #     Dict(:name=> "system", :text=> systemmsg),
-  #     Dict(:name=> "user", :text=> usermsg)
-  #   ]
+    usermsg =
+    """
+    Question: $question
+    Attempts: $potentialSolutionStr
+    """

-  #   # put in model format
-  #   prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
+    _prompt = 
+    [
+      Dict(:name=> "system", :text=> systemmsg),
+      Dict(:name=> "user", :text=> usermsg)
+    ]

-  #   header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"]
-  #   dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
+    # put in model format
+    prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")

-  #   response = text2textInstructLLM(prompt)
+    header = ["Comparison:", "Rationale:", "Selected_response_number:"]
+    dictkey = ["comparison", "rationale", "selected_response_number"]

-  #   # sometime LLM output something like **Comprehension**: which is not expected
-  #   response = replace(response, "**"=>"")
-  #   response = replace(response, "***"=>"")
+    response = text2textInstructLLM(prompt)

-  #   # make sure every header is in the response
-  #   for i in header
-  #     detected = GeneralUtils.detect_keyword(i, response)
-  #     if detected === nothing
-  #       errornote = "Your previous response didn't provide $i"
-  #       errorFlag = true
-  #     end
-  #   end
-  #   if errorFlag
-  #     continue  # skip to the next iteration
-  #   end
+    # sometime LLM output something like **Comprehension**: which is not expected
+    response = replace(response, "**"=>"")
+    response = replace(response, "***"=>"")

-  #   responsedict = GeneralUtils.textToDict(response, header; 
-  #                                           dictKey=dictkey, symbolkey=true)
+    # make sure every header is in the response
+    for i in header
+      detected = GeneralUtils.detect_keyword(i, response)
+      if detected === nothing
+        errornote = "Your previous response didn't provide $i"
+        errorFlag = true
+      end
+    end
+    if errorFlag
+      continue  # skip to the next iteration
+    end
+
+    responsedict = GeneralUtils.textToDict(response, header; 
+                                            dictKey=dictkey, symbolkey=true)
    
-  #   responsedict[:score] = responsedict[:score][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
-  #   try
-  #     responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5
-  #   catch
-  #     continue
-  #   end
+    responsedict[:selected_response_number] = responsedict[:selected_response_number][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
+    try
+      responsedict[:selected_response_number] = parse(Int, responsedict[:selected_response_number]) # convert string "5" into integer 5
+    catch
+      continue
+    end

-  #   accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
+    println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    pprintln(Dict(responsedict))

-  #   if accepted_as_answer ∉ ["Yes", "No"] # [PENDING] add errornote into the prompt
-  #     error("generated accepted_as_answer has wrong format")
-  #   end
-
-  #   # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension)
-  #   state[:evaluation] = "$(responsedict[:trajectory_evaluation]) $(responsedict[:answer_evaluation])" 
-  #   state[:evaluationscore] = responsedict[:score] 
-  #   state[:accepted_as_answer] = responsedict[:accepted_as_answer]
-  #   state[:suggestion] = responsedict[:suggestion]
-
-  #   # mark as terminal state when the answer is achieved
-  #   if accepted_as_answer == "Yes"
-
-  #     # mark the state as terminal state because the evaluation say so.
-  #     state[:isterminal] = true
-
-  #     # evaluation score as reward because different answers hold different value for the user.
-  #     state[:reward] = responsedict[:score]
-  #   end
-  #   println("\n~~~ Evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-  #   pprintln(Dict(responsedict))
-
-  #   return responsedict[:score]
-  # end
-  # error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
+    return responsedict[:selected_response_number]
+  end
+  error("compareState failed to generate an evaluation, Response: \n$response\n<|End of error|>", @__FILE__, ":", @__LINE__, " $(Dates.now())")
 end


--- a/src/util.jl
+++ b/src/util.jl
@@ -1,6 +1,8 @@
 module util

+export makekey

+makekey(key, indice) = Symbol("$(key)_$indice")



--- a/system_prompt_template.jl
+++ b/system_prompt_template.jl
@@ -28,8 +28,7 @@ Default system message template:
  <You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
    Comprehension: State your comprehension about the current situation.
    Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
-    Action_name: (Typically corresponds to the execution of the first step in your plan)
-      Can be one of the following function names:
+    Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names:
      - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
      - CHECKRESOURCES which you can use to check resources
      - IMPLEMENT which you can use to implement the solution
--- a/test/runtest.jl
+++ b/test/runtest.jl
@@ -54,8 +54,9 @@ function text2textInstructLLM(prompt::String; maxattempt=3)
  response = nothing
  for attempts in 1:maxattempt
    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=180, maxattempt=2)
-    response = _response[:response][:text]
-    if response !== nothing
+    payload = _response[:response]
+    if _response[:success] && payload[:text] !== nothing
+      response = _response[:response][:text]
      break
    else
      println("\n<text2textInstructLLM()> attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
@@ -63,6 +64,18 @@ function text2textInstructLLM(prompt::String; maxattempt=3)
      println("</text2textInstructLLM()> attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
      sleep(3)
    end
+    
+    
+    
+    # response = _response[:response][:text]
+    # if response !== nothing
+    #   break
+    # else
+    #   println("\n<text2textInstructLLM()> attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    #   pprintln(outgoingMsg)
+    #   println("</text2textInstructLLM()> attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+    #   sleep(3)
+    # end
  end

  return response
@@ -156,8 +169,14 @@ end
 sessionId = "555"


+# query = "How many German wines do you have?"
+# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
+# selectedState = SQLLLM.compareState(query, highValueStateList, text2textInstructLLM)
+
+
+
 # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?")
-query = "How many German wines do you have?"
+query = "How many French wines from Yiem store under 100 dollars do you have?"
 # query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40"
 # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza"
 # query = "wine_type: white, country: Austria, food to be served with wine: pork"
--- a/test/test_2.jl
+++ b/test/test_2.jl
@@ -1,10 +1,9 @@
-using Revise
-using SQLLLM, LLMMCTS, DataStructures, JSON3
+# using Revise
+# using SQLLLM, LLMMCTS, DataStructures, JSON3

-query = "How many German wines do you have?"
-highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
-
-selectedState = SQLLLM.compareState(query, highValueStateList)
+# query = "How many German wines do you have?"
+# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
+# selectedState = SQLLLM.compareState(query, highValueStateList)