Merge pull request 'update' (#8) from v0.2.8-add_return_msg into v0.2.8

Reviewed-on: #8

Project.toml

@@ -1,7 +1,7 @@
 name = "SQLLLM"
 uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
+version = "0.2.8"
 authors = ["narawat lamaiin <narawat@outlook.com>"]
-version = "0.2.3"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -11,7 +11,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LLMMCTS = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
 LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -24,3 +24,6 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
 Dates = "1.11.0"
+GeneralUtils = "0.4.0 - 0.9.0"
+JSON = "1.6.1"
+LLMMCTS = "0.1.5 - 0.9.0"

README.md

@@ -0,0 +1,82 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

etc.jl

@@ -0,0 +1,22 @@
+d = OrderedDict{String, Any}("1" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT w.winery, w.wine_name, w.wine_id, w.vintage, w.region, w.country, w.wine_type, w.grape, w.serving_temperature, w.sweetness, w.intensity, w.tannin, w.acidity, w.tasting_notes, rw.price, rw.currency, w.image_url FROM wine w JOIN retailer_wine rw ON w.wine_id = rw.wine_id JOIN retailer r ON rw.retailer_id = r.retailer_id WHERE w.wine_name ILIKE 'Montrachet Grand Cru' AND w.winery ILIKE 'Domaine Jacques Prieur' AND w.region ILIKE 'Montrachet' AND r.retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "No records found. Try loosening your search criteria.", "plan" => "The user wants to retrieve specific details about a wine, including its characteristics and pricing at a specific retailer, based on several criteria: wine name, winery, region, retailer name, and retailer ID. I need to join the 'wine', 'retailer_wine', and 'retailer' tables. I will use the wine attributes (name, winery, region) and the retailer ID to filter the results and select all requested fields, using ILIKE for text matching as per guidelines."), "2" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT retailer_name FROM retailer WHERE retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "1) retailer_name: Yiem Wines Ltd\n", "plan" => "The previous query failed with no records found, indicating that the combination of all constraints (wine name, winery, region, and retailer ID) is too restrictive. To proceed, I will first verify the retailer's name using the provided ID. Then, I will execute a new query, joining the 'wine', 'retailer_wine', and 'retailer' tables, filtering only by the wine name ('Montrachet Grand Cru') and the specific retailer ID ('f54eab6b-7650-4448-b009-c53f3efbcc3b'), removing the 'winery' and 'region' constraints to see if any matching records exist at that retailer."))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/llmfunction.jl

@@ -1,10 +1,10 @@
 module llmfunction
 
 export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox,
-  getTableNameFromSQL, extractContent_dataframe, SQLexecution
+  getTableNameFromSQL, extractContent_dataframe, SQLexecution, compareState
 
-using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
+using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
-  DataStructures, StatsBase
+  DataStructures, StatsBase, Dates
 using GeneralUtils, LLMMCTS
 using ..util
 
@@ -36,7 +36,7 @@ julia> result = response[:result]
 # Signature
 """
 function listAllTable_json(executeSQL::Function
-)::NamedTuple{(:result, :success),Tuple{DataFrame,Bool}}
+  )::NamedTuple{(:result, :success),Tuple{DataFrame,Bool}}
 
   sql = """
         SELECT 
@@ -189,7 +189,7 @@ function tableinfo(executeSQL::Function, tablenames::T
     end
   end
   if !isempty(notExistingTable)
-    result = "Error, the following tables does not exist in the database: $(JSON3.write(notExistingTable))"
+    result = "Error, the following tables does not exist in the database: $(JSON.json(notExistingTable))"
     return (result=result,)
   end
 
@@ -287,17 +287,17 @@ function getdata_transition(state::T, args::NamedTuple
   # decisionMaker::Function = args[:decisionMaker]
   # evaluator::Function = args[:evaluator]
   # reflector::Function = args[:reflector]
-  context = args[:context]
+context = args["context"]
-  executeSQL::Function = args[:executeSQL]
+   executeSQL::Function = args["executeSQL"]
-  text2textInstructLLM::Function = args[:text2textInstructLLM]
+   text2textInstructLLM::Function = args["text2textInstructLLM"]
 
-  thought, sql =
+   thought, sql =
-    if state[:code] !== nothing
+     if state["code"] !== nothing
-      result = getdata_decisionMaker(state, context, text2textInstructLLM)
+       result = getdata_decisionMaker(state, context, text2textInstructLLM)
-      result[:thought], result[:code]
+       result["thought"], result["code"]
-    else
+     else
-      nothing, state[:question]
+       nothing, state["question"]
-    end
+     end
 
   # make new state
   newNodeKey = GeneralUtils.uuid4snakecase()
@@ -314,15 +314,15 @@ function getdata_transition(state::T, args::NamedTuple
         isterminal=false)
     end
   println("getdata_transition() 1 ", @__FILE__, " ", @__LINE__)
-  newstate[:code] = sql
+newstate["code"] = sql
-  newstate[:response] = response
+   newstate["response"] = response
-  newstate[:errorexplain] = thought
+   newstate["errorexplain"] = thought
-  newstate[:errormsg] = errormsg
+   newstate["errormsg"] = errormsg
-  newstate[:reward] = reward
+   newstate["reward"] = reward
-  newstate[:isterminal] = isterminal
+   newstate["isterminal"] = isterminal
   if response !== nothing
     extracted = extractContent_dataframe(response, context, text2textInstructLLM)
-    newstate[:response] = extracted
+    newstate["response"] = extracted
   end
   println("getdata_transition() 2 ", @__FILE__, " ", @__LINE__)
   stateevaluation = "None"
@@ -347,8 +347,9 @@ end
 
 # Signature
 """
-function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function
+function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function,
-)::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}}
+  llmFormatName::String
+  )::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}}
 
   Hints = "None"
 
@@ -366,17 +367,14 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
   - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search.
 
   You should then respond to the user with:
-  1) Comprehension: 
+  1) Plan: Step-by-step instructions of how to complete the task.
-      - State your comprehension about the current situation.
-  3) Plan: Step-by-step instructions of how to complete the task.
       - Focus on improving the code from the last round.
       - Do not create any table in the database.
-  4) Code:
+  2) Code:
       - Write new improved code.
       - Do not wrap the code and no comment as it will be executed directly without any modification against the database.
 
   You should only respond in format as described below and nothing more:
-  Comprehension: ...
   Plan: 
     1) ...
     2) ... 
@@ -391,10 +389,10 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
   for attempt in 1:10
     usermsg = """
               Context: 
-              $(context[:mentionedTableInfo])
+              $(context["mentionedTableInfo"])
-              User intention: $(context[:userintention])
+              User intention: $(context["userintention"])
-              Code executed from the last round: $(state[:code])
+              Code executed from the last round: $(state["code"])
-              Execution error: $(state[:errormsg])
+              Execution error: $(state["errormsg"])
               $noise
               $note_flag
               """
@@ -406,21 +404,23 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
       ]
 
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
+    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
     try
-      response = text2textInstructLLM(prompt)
+      response = text2textInstructLLM(prompt, modelsize="medium")
-
+      response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
-      header = ["Comprehension:", "Plan:", "Code:"]
+      think, response = GeneralUtils.extractthink(response)
-      dictkey = ["comprehension", "plan", "code"]
+      
+      header = ["Plan:", "Code:"]
+      dictkey = ["plan", "code"]
 
       responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
+                                            dictKey=dictkey, symbolkey=false)
-      _code = responsedict[:code]
+      _code = responsedict["code"]
       code = strip(_code)
 
       if length(code) < 2
         error("No code available.")
-      elseif code == state[:code]
+      elseif code == state["code"]
         error("generated code is the same as earlier.")
       else
       end
@@ -440,7 +440,7 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
 
       println("\n~~~ getdata_decisionMaker() ", @__FILE__, " ", @__LINE__)
       pprintln(Dict(responsedict))
-      return (thought=responsedict[:comprehension], code=code, success=true, errormsg=nothing)
+      return (thought=responsedict["comprehension"], code=code, success=true, errormsg=nothing)
     catch e
       io = IOBuffer()
       showerror(io, e)
@@ -481,20 +481,9 @@ julia> response = SQLLLM.SQLexecution(executeSQL, sql)
 # Signature
 """
 function SQLexecution(executeSQL::Function, sql::T
-) where {T<:AbstractString}
+  )::NamedTuple where {T<:AbstractString}
 
   try
-    #XXX dummy SQL. use for testing
-    # sql = "SELECT w.wine_name FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.\"food_name\" = 'lamb';"
-    # sql = " SELECT      w.wine_name FROM      wine w JOIN      food f ON f.food_name = 'lamb' JOIN      wine_food wf ON w.wine_id = wf.wine_id AND f.food_id = wf.food_id GROUP BY      w.wine_name ORDER BY      COUNT(DISTINCT w.wine_id) DESC;"
-    # sql = " SELECT COUNT(DISTINCT wf.wine_id) FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.food_name ILIKE '%lamb%'"
-
-    #XXX use for package testing, remove when done
-    # ans = "1.schilfwein zweigelt 2.cabernet sauvignon reserve limited edition"
-    # ans = "There are 1500 wines that can be paired with lamb."
-    # ans = "1500"
-    # return (response=ans, errormsg=nothing, reward=1, isterminal=true)
-
     # add LIMIT to the SQL to prevent loading large data
     sql = strip(sql)
 
@@ -508,39 +497,36 @@ function SQLexecution(executeSQL::Function, sql::T
     else
       sql = sql * ";"
     end
-    println("\n~~~ SQLexecution() SQL: ", @__FILE__, " ", @__LINE__)
-    println(sql)
-
     result = executeSQL(sql)
     df = DataFrame(result)
-
     tablesize = size(df)
     row, column = tablesize
     if row == 0
-      error("The resulting table has 0 row. Possible causes: 1) Your search criteria might be too specific. Relaxing some conditions could yield better results. Remember, you can always refine your search later. 2) There could be a typo in your search query. 3) You might be searching in the wrong place.")
+      return (result_str="No records found. Try loosening your search criteria.", result_raw=nothing, success=true, errormsg=nothing)
     elseif column > 30
-      error("SQL execution failed. An unexpected error occurred. Please try again.")
+      return (result_str="There are more than 30 columns. Please be more specific.", result_raw=df, success=true, errormsg=nothing)
+    else
+      df1 =
+        if row > 2
+          # ramdom row to pick
+          df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df
+        else
+          df
+        end
+      result = GeneralUtils.dfToString(df1)
+      # println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
+      # println(sql)
+      # println(df1)
+      # println("\n")
+      return (result_str=result, result_raw=df1, success=true, errormsg=nothing)
     end
-
-    df1 =
-      if row > 2
-        # ramdom row to pick
-        df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df
-      else
-        df
-      end
-
-    println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
-    println(df1)
-    return (result=df1, success=true, errormsg=nothing)
   catch e
     io = IOBuffer()
     showerror(io, e)
     errorMsg = String(take!(io))
     st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
     println(errorMsg)
-    response = (result=nothing, success=false, errormsg=errorMsg)
+    return (result_str=nothing, result_raw=nothing, success=false, errormsg=errorMsg)
-    return response
   end
 end
 
@@ -559,9 +545,10 @@ end
   - `result::String`
 
 # Signature
-"""
+""" #PENDING
-function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String
+function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String,
-)::String
+  llmFormatName::String
+  )::String
   tablesize = size(df)
   row = tablesize[1]
   column = tablesize[2]
@@ -627,31 +614,34 @@ function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function,
     ]
 
   # put in model format
-  prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
+  prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
   header = ["About_resulting_table:", "Search_summary:"]
   dictkey = ["about_resulting_table", "search_summary"]
 
   for i in 1:5
-    response = text2textInstructLLM(prompt)
+    response = text2textInstructLLM("ramdom_id", prompt)
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
 
-    kw = []
+    # check whether response has all header
-    # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
+    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
-    for keyword in header
+    missingkeys = [k for (k, v) in detected_kw if v === nothing]
-      detected = GeneralUtils.detect_keyword(keyword, response)
+    if !isempty(missingkeys)
-      push!(kw, detected)
+      errornote = "$missingkeys are missing from your previous response"
-    end
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-    if nothing ∈ kw
+      continue
-      println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
-      continue  # try again next loop
+      errornote = "\nYour previous attempt has duplicated points according to the required response format"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
     end
 
     responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
+                                            dictKey=dictkey, symbolkey=false)
 
-    # result = dfstr
     result = 
     """
-    Summary: $(responsedict[:search_summary]) 
+    Summary: $(responsedict["search_summary"]) 
     More details: $dfstr
     """
 
@@ -734,7 +724,9 @@ julia> result = SQLLLM.getTableNameFromSQL(sql, text2textInstructLLM)
 
 # Signature
 """
-function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{String} where {T<:AbstractString}
+function getTableNameFromSQL(sql::T, text2textInstructLLM::Function,
+  llmFormatName::String
+  )::Vector{String} where {T<:AbstractString}
   systemmsg = """
               Extract table name out of the user query.
 
@@ -762,16 +754,17 @@ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{Str
     ]
 
   # put in model format
-  prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
+  prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
   header = ["Table_name:"]
   dictkey = ["table_name"]
 
   for attempt in 1:5
     try
-      response = text2textInstructLLM(prompt)
+      response = text2textInstructLLM(prompt, modelsize="medium")
+      response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
       responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
+                                            dictKey=dictkey, symbolkey=false)
-      response = copy(JSON3.read(responsedict[:table_name]))
+      response = copy(JSON.parse(responsedict["table_name"]))
 
       return response
     catch e
@@ -788,6 +781,188 @@ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{Str
 end
 
 
+""" Compare multiple solution attempts and select the most accurate one.
+
+This function evaluates multiple solution attempts for a given question and determines which attempt 
+provides the most accurate and relevant response. It uses an LLM to analyze and compare the attempts,
+considering their actions and observations.
+
+# Arguments
+- `question::String`
+  The original question or task that was attempted to be solved
+- `highValueStateList::Vector{Dict}`
+  List of states containing different solution attempts and their results
+- `text2textInstructLLM::Function`
+  A function that handles communication to LLM service
+
+# Returns
+- `Integer`
+  The index of the selected best response (1-based indexing)
+
+# Example
+```jldoctest
+julia> 
+```
+
+# Notes
+- The function makes up to 10 attempts to get a valid response from the LLM
+- Each state in highValueStateList should contain a action_history with action_input and observation
+- The LLM evaluates attempts based on accuracy and relevance to the original question
+"""
+function compareState(question::String, highValueStateList::Vector{T},
+  text2textInstructLLM::Function; maxattempt::Integer=10
+  )::Integer where {T<:AbstractDict}
+
+  systemmsg = 
+  """
+  # Your profile:
+  - You are a helpful assistant
+
+  # Situation:
+  - The user has made multiple attempts to solve the question, resulting in various answers
+  
+  # Your mission:
+  - Identify and select the most accurate and relevant response from these multiple results for the user
+  
+  # At each round of conversation, you will be given the following:
+    Question: the question the user is trying to answer
+    Attempt: the user's attempted actions and their corresponding results
+  
+  # You should then respond to the user with the following:
+    1) "comparison", detailed comparison of all results from all attempts from various aspects.
+    2) "rationale", a brief explanation of why the selected response is the most accurate and relevant
+    3) "selected_response_number", the number the selected response in the list of results (e.g., 1, 2, 3, ...)
+
+  # you should only respond in JSON format as described below
+    "comparison": "..."
+    "rationale": "..."
+    "selected_response_number": "..."
+
+  # Here are some examples:
+    Question: "How many German wines do you have?"
+    Attempt 1)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
+    action_result: 100 wines
+    Attempt 2)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
+    action_result: 50 red wines
+    
+    "comparison": "The second attempt counts only German red wines while the first attempt includes all German wines."
+    "rationale": "The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines."
+    "selected_response_number": "1"
+  """
+
+  requiredKeys = ["comparison", "rationale", "selected_response_number"]
+
+  potentialSolution = []
+   includekeys = ["action_name", "action_input", "action_result"]
+   # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
+   for state in highValueStateList
+     action_history = state["action_history"]
+     latestKeys = [i for i in keys(action_history)][end]
+     d = Dict()
+     # get the last action_name, action_input, observation of currentstate
+     for (i,v) in enumerate(includekeys)
+      latest_action = action_history[latestKeys]
+      d[v] = latest_action[v] 
+     end
+     push!(potentialSolution, d)
+   end
+
+  println("\n")
+  @show potentialSolution
+  println("--- ", @__FILE__, @__LINE__)
+
+  """
+  # put potential solutions from potentialSolution into the following form
+  Attempt 1)
+    action_name: 
+    action_input:
+    action_result:
+  Attempt 2)
+    action_name:
+    action_input:
+    action_result:
+  ...
+  """
+
+  potentialSolutionStr = ""
+  for (i, state) in enumerate(potentialSolution)
+    potentialSolutionStr *= "Attempt $i)\n"
+    for k in includekeys
+      potentialSolutionStr *= "$k: $(state[k])\n"
+    end
+  end
+
+  usermsg =
+  """
+  Question: $question
+  $potentialSolutionStr
+  """
+
+  msg = Dict(
+    "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
+    "messages" => [
+      Dict(
+        "role" => "system",
+        "content" => [
+          Dict("type" => "text", "text" => systemmsg),
+        ]
+      ),
+      Dict(
+        "role" => "user",
+        "content" => [        
+          Dict("type" => "text", "text" => usermsg),
+        ]
+      ),
+    ],
+    "temperature" => 0.7
+  )
+
+  for attempt in 1:maxattempt
+    response = text2textInstructLLM("random_id", msg)
+    response = GeneralUtils.clean_json_response(response)
+    response = GeneralUtils.remove_french_accents(response)
+    think, response = GeneralUtils.extractthink(response)
+    response = String(split(response, ", action_result")[1])  # in case LLM generate action_result key which it isn't supposed to
+    response = strip(response)
+    responsedict = nothing
+    try
+      _responsedict = JSON.parse(response)
+      responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
+    catch
+      println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    end
+
+    # check whether all answer's key points are in responsedict
+    ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
+    if !ispass
+      errornote = errormsg
+      println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+      continue
+    end
+
+    try
+      responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
+    catch
+      errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number."
+      println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    end
+
+    # println("\n~~~ compareState() ")
+    # pprintln(Dict(responsedict))
+    # println("---\n", @__FILE__, ":", @__LINE__)
+
+    return responsedict["selected_response_number"]
+  end
+  error("compareState() failed to generate an evaluation, Response: \n$response\n<|End of error|>", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+end
+
+
 
 
 
@@ -824,8 +999,6 @@ end
 
 
 
-
-
 
 
 

src/util.jl

@@ -1,6 +1,8 @@
 module util
 
+export makekey
 
+makekey(key, indice) = "$(key)_$indice"
 
 
 

system_prompt_template.jl

@@ -0,0 +1,160 @@
+"""
+# -------------------------------- Default system message template ------------------------------- #
+
+<Your role>
+  - You are a helpful assistant
+</Your role>
+<Situation>
+  - Describe the current situation
+  Ex. The world use enormous energy from non-sustainable sources. This leads to climate change.
+</Situation>
+<Your vision>
+  - state your vision of how the situation will evolve, what would you want the situation to evolve into
+  Ex. To be the leading innovator in sustainable technology by 2030, transforming global energy systems.
+</Your vision>
+<Your mission>
+  - state the goal
+  Ex. Empowering communities through clean energy solutions to create a sustainable future.
+</Your mission>
+<Your mission's objective includes>
+  - Specific, measurable, and time-bound goals that directly support the mission.
+  Ex. Launch 50 solar-powered water purification systems in 3 regions by 2025.
+</Your mission's objective includes>
+<Your responsibility includes>
+  - state the mini goals that fall under your responsibility
+</Your responsibility includes>
+<Your responsibility does NOT includes>
+  - 
+</Your responsibility does NOT includes>
+<At each round of conversation, you will be given the following information>
+  -
+</At each round of conversation, you will be given the following information>
+<You must follow the following policy>
+  -
+</You must follow the following policy>
+<You should follow the following guidelines>
+  -
+</You should follow the following guidelines>
+<You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+  Comprehension: State your comprehension about the current situation.
+  Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+  Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names:
+    - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
+    - CHECKRESOURCES which you can use to check resources
+    - IMPLEMENT which you can use to implement the solution
+  Action_input: Detail the input for the action.
+</You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+<You should only respond in format as described below>
+  Comprehension: ...
+  Plan: ...
+  Action_name: ...
+  Action_input: ...
+</You should only respond in format as described below>
+<Here are some examples>
+
+</Here are some examples>
+
+Let's begin!
+
+
+
+
+
+
+
+
+
+# ------------------------------------------- Example: ------------------------------------------- #
+
+<Your profile>
+- You are a founder of a tech startup
+</Your profile>
+<Situation>
+- The global rise in bedridden patients, driven by an aging population, presents significant challenges for caregivers. Family members often become primary caretakers, leading to physical and emotional strain. This situation frequently forces caregivers to make difficult choices, including leaving their careers to provide full-time care, which impacts both family finances and personal well-being.
+</Situation>
+<Your vision>
+  - We want to develop a system that can help people with bedridden patients and their families so that they could go on with their lives.
+</Your vision>
+<Your mission>
+  - To create an innovative caregiving support platform that reduces the physical and emotional burden on family caregivers while ensuring quality care for bedridden patients
+</Your mission>
+<Your mission's objectives include>
+  - Develop smart monitoring systems for patient safety
+  - Create automated alert mechanisms for critical situations
+  - Design user-friendly interfaces for remote patient monitoring
+  - Implement AI-driven predictive care recommendations
+  - Build a support network connecting caregivers with healthcare professionals
+  - Establish training modules for family caregivers
+</Your mission's objectives include>
+<Your responsibilities include>
+  - Lead product vision and strategy development
+  - Oversee technical implementation and system architecture
+  - Coordinate with healthcare experts for medical validation
+  - Ensure compliance with healthcare regulations
+  - Manage stakeholder relationships
+  - Drive fundraising and business development
+</Your responsibilities include>
+<At each round of conversation, you will be given the following>
+  Challenges: user's specific caregiving challenges
+  Context: context and severity of the situation
+  Feedback: comments from family caregivers
+  Solutions: potential solution based on immediate and long-term impact
+</At each round of conversation, you will be given the following>
+<You must follow the following guidelines>
+  - Always prioritize patient safety and well-being
+  - Maintain empathy and understanding in all interactions
+  - Focus on practical, implementable solutions
+  - Consider both immediate needs and long-term sustainability
+  - Respect privacy and confidentiality of all stakeholders
+  - Follow healthcare regulations and best practices
+</You must follow the following guidelines>
+<You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+  Comprehension: State your comprehension about the current situation.
+  Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+  Action_name: (Typically corresponds to the execution of the first step in your plan)
+    Can be one of the following function names:
+    - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
+    - CHECKRESOURCES which you can use to check resources
+    - IMPLEMENT which you can use to implement the solution
+  Action_input: Detail the input for the action.
+</You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+<You should only respond in format as described below>
+  Comprehension: ...
+  Plan: ...
+  Action_name: ...
+  Action_input: ...
+</You should only respond in format as described below>
+<Here are some examples>
+  Example 1:
+  Challenges: "My mother needs constant monitoring at night, but I'm exhausted from lack of sleep."
+  Context: Elderly patient with dementia, requires 24/7 supervision
+  Feedback: "Need urgent solution for night monitoring"
+  Solutions: Smart monitoring system with motion sensors and alerts
+
+  Comprehension: The caregiver is experiencing severe sleep deprivation due to nighttime monitoring requirements
+  Plan: 
+    1. Assess current monitoring needs
+    2. Propose smart monitoring system installation
+    3. Set up emergency alert system
+    4. Train family on system usage
+  Action_name: CHATBOX
+  Action_input: Discuss specific nighttime behaviors and incidents to determine optimal sensor placement and alert thresholds
+
+  Example 2:
+  Challenges: "Managing medication schedules is becoming overwhelming"
+  Context: Patient on multiple medications with complex timing requirements
+  Feedback: "Need help with medication management"
+  Solutions: Automated medication reminder and tracking system
+
+  Comprehension: Caregiver struggling with complex medication management tasks
+  Plan:
+    1. Review current medication schedule
+    2. Implement automated reminder system
+    3. Set up medication tracking log
+    4. Connect with pharmacy for refill automation
+  Action_name: IMPLEMENT
+  Action_input: Deploy medication management module with smart alerts and compliance tracking
+</Here are some examples>
+
+Let's begin!
+"""

test/Manifest.toml

@@ -0,0 +1,41 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.11.4"
+manifest_format = "2.0"
+project_hash = "71d91126b5a1fb1020e1098d9d492de2a4438fd2"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+version = "1.11.0"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+version = "1.11.0"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+version = "1.11.0"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+version = "1.11.0"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+version = "1.11.0"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+version = "1.11.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+version = "1.11.0"

test/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/runtests.jl

@@ -1,9 +1,9 @@
 using Revise
-using LibPQ, Dates, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
+using LibPQ, Dates, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
 using GeneralUtils, SQLLLM
 
 
-config = copy(JSON3.read("/appfolder/mountvolume/appdata/config.json"))
+config = JSON.parse("/appfolder/app/dev/YiemAgent/test/config.json")
 
 function executeSQL(sql::T) where {T<:AbstractString}
   host = config[:externalservice][:wineDB][:host]
@@ -29,13 +29,19 @@ function executeSQLVectorDB(sql)
   return result
 end
 
-function text2textInstructLLM(prompt::String; maxattempt=3)
+function text2textInstructLLM(prompt::String; maxattempt::Integer=3, modelsize::String="medium",
+    senderId=GeneralUtils.uuid4snakecase(), timeout=180,
+    llmkwargs=Dict(
+      :num_ctx => 32768,
+      :temperature => 0.5,
+    )
+  )
   msgMeta = GeneralUtils.generate_msgMeta(
     config[:externalservice][:loadbalancer][:mqtttopic];
     msgPurpose="inference",
     senderName="yiemagent",
-    senderId=sessionId,
+    senderId=senderId,
-    receiverName="text2textinstruct_small",
+    receiverName="text2textinstruct_$modelsize",
     mqttBrokerAddress=config[:mqttServerInfo][:broker],
     mqttBrokerPort=config[:mqttServerInfo][:port],
   )
@@ -44,18 +50,16 @@ function text2textInstructLLM(prompt::String; maxattempt=3)
     :msgMeta => msgMeta,
     :payload => Dict(
       :text => prompt,
-      :kwargs => Dict(
+      :kwargs => llmkwargs
-        :num_ctx => 16384,
-        :temperature => 0.2,
-      )
     )
   )
 
   response = nothing
   for attempts in 1:maxattempt
-    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=180, maxattempt=2)
+    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=timeout, maxattempt=maxattempt)
-    response = _response[:response][:text]
+    payload = _response[:response]
-    if response !== nothing
+    if _response[:success] && payload[:text] !== nothing
+      response = _response[:response][:text]
       break
     else
       println("\n<text2textInstructLLM()> attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
@@ -75,7 +79,7 @@ function getEmbedding(text::T) where {T<:AbstractString}
     msgPurpose="embedding",
     senderName="yiemagent",
     senderId=sessionId,
-    receiverName="text2textinstruct_small",
+    receiverName="textembedding",
     mqttBrokerAddress=config[:mqttServerInfo][:broker],
     mqttBrokerPort=config[:mqttServerInfo][:port],
   )
@@ -86,7 +90,8 @@ function getEmbedding(text::T) where {T<:AbstractString}
       :text => [text] # must be a vector of string
     )
   )
-  response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
+
+  response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120, maxattempt=3)
   embedding = response[:response][:embeddings]
   return embedding
 end
@@ -107,12 +112,12 @@ function findSimilarTextFromVectorDB(text::T1, tablename::T2, embeddingColumnNam
   return df
 end
 
-
 function similarSQLVectorDB(query; maxdistance::Integer=100)
   tablename = "sqlllm_decision_repository"
   # get embedding of the query
   df = findSimilarTextFromVectorDB(query, tablename, 
                                   "function_input_embedding", executeSQLVectorDB)
+  # println(df[1, [:id, :function_output]])
   row, col = size(df)
   distance = row == 0 ? Inf : df[1, :distance]  
   # distance = 100  # CHANGE this is for testing only
@@ -129,11 +134,10 @@ function similarSQLVectorDB(query; maxdistance::Integer=100)
   end
 end
 
-
 function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString}
   tablename = "sqlllm_decision_repository"
   # get embedding of the query
-  # query = state[:thoughtHistory][:question]
+  # query = state[:action_history][:question]
   df = findSimilarTextFromVectorDB(query, tablename, 
                                   "function_input_embedding", executeSQLVectorDB)
   row, col = size(df)
@@ -153,12 +157,77 @@ function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1
   end
 end
 
-sessionId = "555"
+
+function similarSommelierDecision(recentevents::T1; maxdistance::Integer=3
+  )::Union{AbstractDict, Nothing} where {T1<:AbstractString}
+  tablename = "sommelier_decision_repository"
+  # find similar
+  println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__)
+  df = findSimilarTextFromVectorDB(recentevents, tablename, 
+                                  "function_input_embedding", executeSQLVectorDB)
+  row, col = size(df)
+  distance = row == 0 ? Inf : df[1, :distance]
+  if row != 0 && distance < maxdistance
+    # if there is usable decision, return it.
+    rowid = df[1, :id]
+    println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
+    output_b64 = df[1, :function_output_base64]   # pick the closest match
+    _output_str = String(base64decode(output_b64))
+    output = copy(JSON.parse(_output_str))
+    return output
+  else
+    println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
+    return nothing
+  end
+end
+
+
+function insertSommelierDecision(recentevents::T1, decision::T2; maxdistance::Integer=5
+  ) where {T1<:AbstractString, T2<:AbstractDict}
+  tablename = "sommelier_decision_repository"
+  # find similar
+  df = findSimilarTextFromVectorDB(recentevents, tablename, 
+                                  "function_input_embedding", executeSQLVectorDB)
+  row, col = size(df)
+  distance = row == 0 ? Inf : df[1, :distance]
+  if row == 0 || distance > maxdistance # no close enough SQL stored in the database
+    recentevents_embedding = getEmbedding(recentevents)[1]
+    recentevents = replace(recentevents, "'" => "")
+    decision_json = JSON.json(decision)
+    decision_base64 = base64encode(decision_json)
+    decision = replace(decision_json, "'" => "")
+    
+    sql = """
+          INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding');
+          """
+    println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__)
+    println(sql)
+    _ = executeSQLVectorDB(sql)
+  else
+    println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__)
+  end
+end
+
+sessionId = GeneralUtils.uuid4snakecase()
+d = Dict(:id => sessionId)
+filepath = "/appfolder/app/sessionid.json"
+open(filepath, "w") do io
+  JSON.pretty(io, d)
+end
+
+
+
+
+# query = "How many German wines do you have?"
+# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
+# selectedState = SQLLLM.compareState(query, highValueStateList, text2textInstructLLM)
+
 
 
 # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?")
-query = "How many French wines do you have?"
+# query = "How many French wines from Yiem store under 100 dollars do you have?"
 # query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40"
+query = "from Yiem retailer, red wine from France. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5"
 # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza"
 # query = "wine_type: white, country: Austria, food to be served with wine: pork"
 # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"
@@ -283,8 +352,8 @@ SELECT * FROM wine WHERE wine_type = 'red' AND country = 'France' AND sweetness
 #   :evaluation =>
 #       "The user's question is to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. The thought is correct in identifying the conditions needed to filter the wine table. The action taken is to execute a SQL query to retrieve the desired data, which is also correct. The observation provides a search summary and two search results that match the user's question. Each result includes details about the wine such as ID, name, brand, manufacturer, region, country, type, grape variety, serving temperature, intensity, sweetness, tannin, and acidity.",
 #   :accepted_as_answer => "Yes",
-#   :thoughtHistory =>
+#   :action_history =>
-#       OrderedDict{Symbol, Any}(:question => "Search the database for wine_type: white, country: France, sweetness: 1", :thought_1 => "The user wants to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. To achieve this, we need to filter the wine table based on these conditions.", :action_name_1 => "GETDATA", :action_input_1 => "SELECT * FROM wine WHERE wine.wine_type = 'white' AND wine.country = 'France' AND wine.sweetness = 1;", :observation_1 => "\"Search summary: The resulting table represents wines.\\nSearch result: 1) wine_id: 5b6b6df9-d87c-4f33-8995-7249c2ecc917, wine_name: corton-charlemagne grand cru, brand: domaine des croix, manufacturer: domaine des croix, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n2) wine_id: 1ad27d16-ef64-4907-acf1-40631630c143, wine_name: puligny-montrachet 1er cru 'les demoiselles', brand: amiot guy, manufacturer: amiot guy, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n\\n\""),
+#       OrderedDict{String, Any}("question" => "Search the database for wine_type: white, country: France, sweetness: 1", "thought_1" => "The user wants to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. To achieve this, we need to filter the wine table based on these conditions.", "action_name_1" => "GETDATA", "action_input_1" => "SELECT * FROM wine WHERE wine.wine_type = 'white' AND wine.country = 'France' AND wine.sweetness = 1;", "observation_1" => "\"Search summary: The resulting table represents wines.\\nSearch result: 1) wine_id: 5b6b6df9-d87c-4f33-8995-7249c2ecc917, wine_name: corton-charlemagne grand cru, brand: domaine des croix, manufacturer: domaine des croix, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n2) wine_id: 1ad27d16-ef64-4907-acf1-40631630c143, wine_name: puligny-montrachet 1er cru 'les demoiselles', brand: amiot guy, manufacturer: amiot guy, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n\\n\""),
 #   :evaluationscore => 9,
 #   :select => nothing,
 #   :suggestion => "None")

test/test_1.jl

@@ -1,5 +1,5 @@
 using Revise
-using LibPQ, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
+using LibPQ, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
 using SQLLLM, GeneralUtils
 
 
@@ -39,7 +39,7 @@ df = DataFrame(result)
 
 
 
-config = copy(JSON3.read("config.json"))
+config = copy(JSON.parse("config.json"))
 
 msgMeta = GeneralUtils.generate_msgMeta(
     config[:externalservice][:text2textinstruct][:mqtttopic];

test/test_2.jl

@@ -1,16 +1,70 @@
-using Revise
+# using Revise
+# using SQLLLM, LLMMCTS, DataStructures, JSON
+
+# query = "How many German wines do you have?"
+# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
+# selectedState = SQLLLM.compareState(query, highValueStateList)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-function testf(a)::NamedTuple{(:a, :b), Tuple{Union{Nothing, Int}, Int}}
-  if a == 1
-    return (a=nothing, b=5)
-  else
-    return (a=5, b=5)
-  end
-end
-
-
-q = testf(1)
-w = testf(2)
 
 
 

testfile.csv

@@ -1,8 +0,0 @@
-table_name,comment
-customer,"The customer table stores information about customers. It includes details such as first name, last name, display name, username, password, gender, country, telephone number, email, birthdate, additional_search_term, other attributes (in JSON format) and a description."
-wine,"The wine table stores information about different wines. It includes details namely id, name, brand, manufacturer, region, country, wine_type, grape_variety, serving_temperature, intensity, sweetness, tannin, acidity, fizziness, additional_search_term, other attributes (in JSON format) and a description."
-wine_food,"The wine_food table represents the association between wines and food items. It establishes a many-to-many relationship, allowing us to link specific wines with various food items."
-food,"The food table represents various food items. It stores information related to food names, country of origin, taste attributes (spiciness, sweetness, sourness, savoriness, and bitterness), serving temperature, additional_search_term, other attributes (in JSON format) and a description."
-retailer,"The retailer table stores information about different retailers. It includes details related to retailer names, usernames, passwords, addresses, contact persons, telephone numbers, email addresses, additional_search_term, other attributes (in JSON format) and a description."
-retailer_wine,"The retailer_wine table represents the relationship between retailers and wines. It stores information about the wines available from which retailers, including vintage, their price, and the currency."
-retailer_food,"The retailer_food table represents the relationship between retailers and food items. It stores information about the food items available from which retailers, including their price and the currency."