Merge pull request 'update' (#8) from v0.2.8-add_return_msg into v0.2.8

Reviewed-on: #8

Project.toml

@@ -1,7 +1,7 @@
 name = "SQLLLM"
 uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
+version = "0.2.8"
 authors = ["narawat lamaiin <narawat@outlook.com>"]
-version = "0.2.4"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -11,7 +11,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LLMMCTS = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
 LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -24,3 +24,6 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
 Dates = "1.11.0"
+GeneralUtils = "0.4.0 - 0.9.0"
+JSON = "1.6.1"
+LLMMCTS = "0.1.5 - 0.9.0"

README.md

@@ -0,0 +1,82 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

etc.jl

@@ -0,0 +1,22 @@
+d = OrderedDict{String, Any}("1" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT w.winery, w.wine_name, w.wine_id, w.vintage, w.region, w.country, w.wine_type, w.grape, w.serving_temperature, w.sweetness, w.intensity, w.tannin, w.acidity, w.tasting_notes, rw.price, rw.currency, w.image_url FROM wine w JOIN retailer_wine rw ON w.wine_id = rw.wine_id JOIN retailer r ON rw.retailer_id = r.retailer_id WHERE w.wine_name ILIKE 'Montrachet Grand Cru' AND w.winery ILIKE 'Domaine Jacques Prieur' AND w.region ILIKE 'Montrachet' AND r.retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "No records found. Try loosening your search criteria.", "plan" => "The user wants to retrieve specific details about a wine, including its characteristics and pricing at a specific retailer, based on several criteria: wine name, winery, region, retailer name, and retailer ID. I need to join the 'wine', 'retailer_wine', and 'retailer' tables. I will use the wine attributes (name, winery, region) and the retailer ID to filter the results and select all requested fields, using ILIKE for text matching as per guidelines."), "2" => Dict{String, Any}("action_name" => "RUNSQL", "action_input" => "SELECT retailer_name FROM retailer WHERE retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b';", "action_result" => "1) retailer_name: Yiem Wines Ltd\n", "plan" => "The previous query failed with no records found, indicating that the combination of all constraints (wine name, winery, region, and retailer ID) is too restrictive. To proceed, I will first verify the retailer's name using the provided ID. Then, I will execute a new query, joining the 'wine', 'retailer_wine', and 'retailer' tables, filtering only by the wine name ('Montrachet Grand Cru') and the specific retailer ID ('f54eab6b-7650-4448-b009-c53f3efbcc3b'), removing the 'winery' and 'region' constraints to see if any matching records exist at that retailer."))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/llmfunction.jl

@@ -3,7 +3,7 @@ module llmfunction
 export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox,
   getTableNameFromSQL, extractContent_dataframe, SQLexecution, compareState
 
-using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
+using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
   DataStructures, StatsBase, Dates
 using GeneralUtils, LLMMCTS
 using ..util
@@ -189,7 +189,7 @@ function tableinfo(executeSQL::Function, tablenames::T
     end
   end
   if !isempty(notExistingTable)
-    result = "Error, the following tables does not exist in the database: $(JSON3.write(notExistingTable))"
+    result = "Error, the following tables does not exist in the database: $(JSON.json(notExistingTable))"
     return (result=result,)
   end
 
@@ -287,17 +287,17 @@ function getdata_transition(state::T, args::NamedTuple
   # decisionMaker::Function = args[:decisionMaker]
   # evaluator::Function = args[:evaluator]
   # reflector::Function = args[:reflector]
-  context = args[:context]
-  executeSQL::Function = args[:executeSQL]
-  text2textInstructLLM::Function = args[:text2textInstructLLM]
+context = args["context"]
+   executeSQL::Function = args["executeSQL"]
+   text2textInstructLLM::Function = args["text2textInstructLLM"]
 
-  thought, sql =
-    if state[:code] !== nothing
-      result = getdata_decisionMaker(state, context, text2textInstructLLM)
-      result[:thought], result[:code]
-    else
-      nothing, state[:question]
-    end
+   thought, sql =
+     if state["code"] !== nothing
+       result = getdata_decisionMaker(state, context, text2textInstructLLM)
+       result["thought"], result["code"]
+     else
+       nothing, state["question"]
+     end
 
   # make new state
   newNodeKey = GeneralUtils.uuid4snakecase()
@@ -314,15 +314,15 @@ function getdata_transition(state::T, args::NamedTuple
         isterminal=false)
     end
   println("getdata_transition() 1 ", @__FILE__, " ", @__LINE__)
-  newstate[:code] = sql
-  newstate[:response] = response
-  newstate[:errorexplain] = thought
-  newstate[:errormsg] = errormsg
-  newstate[:reward] = reward
-  newstate[:isterminal] = isterminal
+newstate["code"] = sql
+   newstate["response"] = response
+   newstate["errorexplain"] = thought
+   newstate["errormsg"] = errormsg
+   newstate["reward"] = reward
+   newstate["isterminal"] = isterminal
   if response !== nothing
     extracted = extractContent_dataframe(response, context, text2textInstructLLM)
-    newstate[:response] = extracted
+    newstate["response"] = extracted
   end
   println("getdata_transition() 2 ", @__FILE__, " ", @__LINE__)
   stateevaluation = "None"
@@ -347,8 +347,9 @@ end
 
 # Signature
 """
-function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function
-)::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}}
+function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function,
+  llmFormatName::String
+  )::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}}
 
   Hints = "None"
 
@@ -366,17 +367,14 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
   - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search.
 
   You should then respond to the user with:
-  1) Comprehension: 
-      - State your comprehension about the current situation.
-  3) Plan: Step-by-step instructions of how to complete the task.
+  1) Plan: Step-by-step instructions of how to complete the task.
       - Focus on improving the code from the last round.
       - Do not create any table in the database.
-  4) Code:
+  2) Code:
       - Write new improved code.
       - Do not wrap the code and no comment as it will be executed directly without any modification against the database.
 
   You should only respond in format as described below and nothing more:
-  Comprehension: ...
   Plan: 
     1) ...
     2) ... 
@@ -391,10 +389,10 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
   for attempt in 1:10
     usermsg = """
               Context: 
-              $(context[:mentionedTableInfo])
-              User intention: $(context[:userintention])
-              Code executed from the last round: $(state[:code])
-              Execution error: $(state[:errormsg])
+              $(context["mentionedTableInfo"])
+              User intention: $(context["userintention"])
+              Code executed from the last round: $(state["code"])
+              Execution error: $(state["errormsg"])
               $noise
               $note_flag
               """
@@ -406,22 +404,23 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
       ]
 
     # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+    prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
     try
       response = text2textInstructLLM(prompt, modelsize="medium")
-      response = GeneralUtils.deFormatLLMtext(response, "granite3")
-
-      header = ["Comprehension:", "Plan:", "Code:"]
-      dictkey = ["comprehension", "plan", "code"]
+      response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+      think, response = GeneralUtils.extractthink(response)
+      
+      header = ["Plan:", "Code:"]
+      dictkey = ["plan", "code"]
 
       responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
-      _code = responsedict[:code]
+                                            dictKey=dictkey, symbolkey=false)
+      _code = responsedict["code"]
       code = strip(_code)
 
       if length(code) < 2
         error("No code available.")
-      elseif code == state[:code]
+      elseif code == state["code"]
         error("generated code is the same as earlier.")
       else
       end
@@ -441,7 +440,7 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM:
 
       println("\n~~~ getdata_decisionMaker() ", @__FILE__, " ", @__LINE__)
       pprintln(Dict(responsedict))
-      return (thought=responsedict[:comprehension], code=code, success=true, errormsg=nothing)
+      return (thought=responsedict["comprehension"], code=code, success=true, errormsg=nothing)
     catch e
       io = IOBuffer()
       showerror(io, e)
@@ -482,20 +481,9 @@ julia> response = SQLLLM.SQLexecution(executeSQL, sql)
 # Signature
 """
 function SQLexecution(executeSQL::Function, sql::T
-) where {T<:AbstractString}
+  )::NamedTuple where {T<:AbstractString}
 
   try
-    #XXX dummy SQL. use for testing
-    # sql = "SELECT w.wine_name FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.\"food_name\" = 'lamb';"
-    # sql = " SELECT      w.wine_name FROM      wine w JOIN      food f ON f.food_name = 'lamb' JOIN      wine_food wf ON w.wine_id = wf.wine_id AND f.food_id = wf.food_id GROUP BY      w.wine_name ORDER BY      COUNT(DISTINCT w.wine_id) DESC;"
-    # sql = " SELECT COUNT(DISTINCT wf.wine_id) FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.food_name ILIKE '%lamb%'"
-
-    #XXX use for package testing, remove when done
-    # ans = "1.schilfwein zweigelt 2.cabernet sauvignon reserve limited edition"
-    # ans = "There are 1500 wines that can be paired with lamb."
-    # ans = "1500"
-    # return (response=ans, errormsg=nothing, reward=1, isterminal=true)
-
     # add LIMIT to the SQL to prevent loading large data
     sql = strip(sql)
 
@@ -509,39 +497,36 @@ function SQLexecution(executeSQL::Function, sql::T
     else
       sql = sql * ";"
     end
-    println("\n~~~ SQLexecution() SQL: ", @__FILE__, " ", @__LINE__)
-    println(sql)
-
     result = executeSQL(sql)
     df = DataFrame(result)
-
     tablesize = size(df)
     row, column = tablesize
     if row == 0
-      error("The resulting table has 0 row. Possible causes: 1) Your search criteria might be too specific. Relaxing some conditions could yield better results. Remember, you can always refine your search later. 2) There could be a typo in your search query. 3) You might be searching in the wrong place.")
+      return (result_str="No records found. Try loosening your search criteria.", result_raw=nothing, success=true, errormsg=nothing)
     elseif column > 30
-      error("SQL execution failed. An unexpected error occurred. Please try again.")
+      return (result_str="There are more than 30 columns. Please be more specific.", result_raw=df, success=true, errormsg=nothing)
+    else
+      df1 =
+        if row > 2
+          # ramdom row to pick
+          df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df
+        else
+          df
+        end
+      result = GeneralUtils.dfToString(df1)
+      # println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
+      # println(sql)
+      # println(df1)
+      # println("\n")
+      return (result_str=result, result_raw=df1, success=true, errormsg=nothing)
     end
-
-    df1 =
-      if row > 2
-        # ramdom row to pick
-        df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df
-      else
-        df
-      end
-
-    println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
-    println(df1)
-    return (result=df1, success=true, errormsg=nothing)
   catch e
     io = IOBuffer()
     showerror(io, e)
     errorMsg = String(take!(io))
     st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
     println(errorMsg)
-    response = (result=nothing, success=false, errormsg=errorMsg)
-    return response
+    return (result_str=nothing, result_raw=nothing, success=false, errormsg=errorMsg)
   end
 end
 
@@ -560,9 +545,10 @@ end
   - `result::String`
 
 # Signature
-"""
-function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String
-)::String
+""" #PENDING
+function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String,
+  llmFormatName::String
+  )::String
   tablesize = size(df)
   row = tablesize[1]
   column = tablesize[2]
@@ -628,32 +614,34 @@ function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function,
     ]
 
   # put in model format
-  prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+  prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
   header = ["About_resulting_table:", "Search_summary:"]
   dictkey = ["about_resulting_table", "search_summary"]
 
   for i in 1:5
-    response = text2textInstructLLM(prompt, modelsize="medium")
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
+    response = text2textInstructLLM("ramdom_id", prompt)
+    response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
+    think, response = GeneralUtils.extractthink(response)
 
-    kw = []
-    # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
-    for keyword in header
-      detected = GeneralUtils.detect_keyword(keyword, response)
-      push!(kw, detected)
-    end
-    if nothing ∈ kw
-      println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue  # try again next loop
+    # check whether response has all header
+    detected_kw = GeneralUtils.detectKeywordVariation(header, response)
+    missingkeys = [k for (k, v) in detected_kw if v === nothing]
+    if !isempty(missingkeys)
+      errornote = "$missingkeys are missing from your previous response"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    elseif sum([length(i) for i in values(detected_kw)]) > length(header)
+      errornote = "\nYour previous attempt has duplicated points according to the required response format"
+      println("\nERROR SQLLLM extractContent_dataframe() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
     end
 
     responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
+                                            dictKey=dictkey, symbolkey=false)
 
-    # result = dfstr
     result = 
     """
-    Summary: $(responsedict[:search_summary]) 
+    Summary: $(responsedict["search_summary"]) 
     More details: $dfstr
     """
 
@@ -736,7 +724,9 @@ julia> result = SQLLLM.getTableNameFromSQL(sql, text2textInstructLLM)
 
 # Signature
 """
-function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{String} where {T<:AbstractString}
+function getTableNameFromSQL(sql::T, text2textInstructLLM::Function,
+  llmFormatName::String
+  )::Vector{String} where {T<:AbstractString}
   systemmsg = """
               Extract table name out of the user query.
 
@@ -764,17 +754,17 @@ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{Str
     ]
 
   # put in model format
-  prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
+  prompt = GeneralUtils.formatLLMtext(_prompt, llmFormatName)
   header = ["Table_name:"]
   dictkey = ["table_name"]
 
   for attempt in 1:5
     try
       response = text2textInstructLLM(prompt, modelsize="medium")
-      response = GeneralUtils.deFormatLLMtext(response, "granite3")
+      response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
       responsedict = GeneralUtils.textToDict(response, header; 
-                                            dictKey=dictkey, symbolkey=true)
-      response = copy(JSON3.read(responsedict[:table_name]))
+                                            dictKey=dictkey, symbolkey=false)
+      response = copy(JSON.parse(responsedict["table_name"]))
 
       return response
     catch e
@@ -816,148 +806,158 @@ julia>
 
 # Notes
 - The function makes up to 10 attempts to get a valid response from the LLM
-- Each state in highValueStateList should contain a thoughtHistory with action_input and observation
+- Each state in highValueStateList should contain a action_history with action_input and observation
 - The LLM evaluates attempts based on accuracy and relevance to the original question
 """
 function compareState(question::String, highValueStateList::Vector{T},
-                      text2textInstructLLM::Function)::Integer where {T<:AbstractDict}
+  text2textInstructLLM::Function; maxattempt::Integer=10
+  )::Integer where {T<:AbstractDict}
 
   systemmsg = 
   """
-    <Your profile>
-    - You are a helpful assistant
-    </Your profile>
-    <Situation>
-    The user has made multiple attempts to solve the question, resulting in various answers
-    <Your mission>
-    - Identify and select the most accurate and relevant response from these multiple results for the user
-    </Your mission>
-    <At each round of conversation, you will be given the following>
-      Question: the question the user is trying to answer
-      Attempt: the user's attempted actions and their corresponding results
-    </At each round of conversation, you will be given the following>
-    <You should then respond to the user with the following>
-      Comparison: a comparison of all results from all attempts
-      Rationale: a brief explanation of why the selected response is the most accurate and relevant
-      Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
-    </You should then respond to the user with the following>
-    <You should only respond in format as described below>
-      Comparison: ...
-      Rationale: ...
-      Selected_response_number: ...
-    </You should only respond in format as described below>
-    <Here are some examples>
-    User's question: "How many German wines do you have?"
-    Attempt 1:
-    Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
-    Result: 100 wines
-    Attempt 2:
-    Action: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
-    Result: 50 red wines
-    Comparison: The second attempt counts only German red wines while the first attempt includes all German wines.
-    Rationale: The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines.
-    Selected_response_number:1
-    </Here are some examples>
+  # Your profile:
+  - You are a helpful assistant
 
-    Let's begin!
+  # Situation:
+  - The user has made multiple attempts to solve the question, resulting in various answers
+  
+  # Your mission:
+  - Identify and select the most accurate and relevant response from these multiple results for the user
+  
+  # At each round of conversation, you will be given the following:
+    Question: the question the user is trying to answer
+    Attempt: the user's attempted actions and their corresponding results
+  
+  # You should then respond to the user with the following:
+    1) "comparison", detailed comparison of all results from all attempts from various aspects.
+    2) "rationale", a brief explanation of why the selected response is the most accurate and relevant
+    3) "selected_response_number", the number the selected response in the list of results (e.g., 1, 2, 3, ...)
+
+  # you should only respond in JSON format as described below
+    "comparison": "..."
+    "rationale": "..."
+    "selected_response_number": "..."
+
+  # Here are some examples:
+    Question: "How many German wines do you have?"
+    Attempt 1)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany'
+    action_result: 100 wines
+    Attempt 2)
+    action_name: RUNSQL
+    action_input: SELECT COUNT(*) FROM wines WHERE country = 'Germany' AND type = 'Red'
+    action_result: 50 red wines
+    
+    "comparison": "The second attempt counts only German red wines while the first attempt includes all German wines."
+    "rationale": "The user is asking for the number of German wines without specifying a type, so the most accurate response is the first attempt because it includes all German wines."
+    "selected_response_number": "1"
   """
 
+  requiredKeys = ["comparison", "rationale", "selected_response_number"]
+
   potentialSolution = []
-  keys = [:action_input, :observation]
-  # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
-  for state in highValueStateList
-    thoughtHistory = state[:thoughtHistory]
-    _, currentstate_latestIndice = 
-      GeneralUtils.findHighestIndexKey(thoughtHistory, keys[1])
-    latestKeys = makekey.(keys, currentstate_latestIndice)
-    d = Dict()
-    # get the last action_name, action_input, observation of currentstate
-    for (i,v) in enumerate(keys)
-      d[v] = thoughtHistory[latestKeys[i]]
-    end
-    push!(potentialSolution, d)
-  end
+   includekeys = ["action_name", "action_input", "action_result"]
+   # extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
+   for state in highValueStateList
+     action_history = state["action_history"]
+     latestKeys = [i for i in keys(action_history)][end]
+     d = Dict()
+     # get the last action_name, action_input, observation of currentstate
+     for (i,v) in enumerate(includekeys)
+      latest_action = action_history[latestKeys]
+      d[v] = latest_action[v] 
+     end
+     push!(potentialSolution, d)
+   end
+
+  println("\n")
+  @show potentialSolution
+  println("--- ", @__FILE__, @__LINE__)
 
   """
   # put potential solutions from potentialSolution into the following form
-  Attempt 1
-  action_name: 
-  action_input:
-  observation:
-  Attempt 2
-  action_name: 
-  action_input:
-  observation:
+  Attempt 1)
+    action_name: 
+    action_input:
+    action_result:
+  Attempt 2)
+    action_name:
+    action_input:
+    action_result:
   ...
   """
+
   potentialSolutionStr = ""
   for (i, state) in enumerate(potentialSolution)
-    potentialSolutionStr *= "Attempt $i\n"
-    for k in keys
+    potentialSolutionStr *= "Attempt $i)\n"
+    for k in includekeys
       potentialSolutionStr *= "$k: $(state[k])\n"
-      println("")
     end
   end
 
-  errornote = ""
+  usermsg =
+  """
+  Question: $question
+  $potentialSolutionStr
+  """
 
-  for attempt in 1:10
-    errorFlag = false
+  msg = Dict(
+    "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
+    "messages" => [
+      Dict(
+        "role" => "system",
+        "content" => [
+          Dict("type" => "text", "text" => systemmsg),
+        ]
+      ),
+      Dict(
+        "role" => "user",
+        "content" => [        
+          Dict("type" => "text", "text" => usermsg),
+        ]
+      ),
+    ],
+    "temperature" => 0.7
+  )
 
-    usermsg =
-    """
-    Question: $question
-    Attempts: $potentialSolutionStr
-    P.S. $errornote
-    """
-
-    _prompt = 
-    [
-      Dict(:name=> "system", :text=> systemmsg),
-      Dict(:name=> "user", :text=> usermsg)
-    ]
-
-    # put in model format
-    prompt = GeneralUtils.formatLLMtext(_prompt, "granite3")
-
-    header = ["Comparison:", "Rationale:", "Selected_response_number:"]
-    dictkey = ["comparison", "rationale", "selected_response_number"]
-
-    response = text2textInstructLLM(prompt, modelsize="medium")
-
-    # sometime LLM output something like **Comprehension**: which is not expected
-    response = replace(response, "**"=>"")
-    response = replace(response, "***"=>"")
-    response = GeneralUtils.deFormatLLMtext(response, "granite3")
-
-    # make sure every header is in the response
-    for i in header
-      detected = GeneralUtils.detect_keyword(i, response)
-      if detected === nothing
-        errornote = "Your previous attempt didn't provide $i"
-        errorFlag = true
-      end
-    end
-    if errorFlag  
-      println("\nERROR SQLLLM compareState() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-      continue  # skip to the next iteration
-    end
-
-    responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=true)
-
-    responsedict[:selected_response_number] = responsedict[:selected_response_number][1]  # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
+  for attempt in 1:maxattempt
+    response = text2textInstructLLM("random_id", msg)
+    response = GeneralUtils.clean_json_response(response)
+    response = GeneralUtils.remove_french_accents(response)
+    think, response = GeneralUtils.extractthink(response)
+    response = String(split(response, ", action_result")[1])  # in case LLM generate action_result key which it isn't supposed to
+    response = strip(response)
+    responsedict = nothing
     try
-      responsedict[:selected_response_number] = parse(Int, responsedict[:selected_response_number]) # convert string "5" into integer 5
+      _responsedict = JSON.parse(response)
+      responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
+    catch
+      println("\nERROR SQLLLM evaluator() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
+      continue
+    end
+
+    # check whether all answer's key points are in responsedict
+    ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
+    if !ispass
+      errornote = errormsg
+      println("\nERROR SQLLLM evaluator() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
+      continue
+    end
+
+    try
+      responsedict["selected_response_number"] = parse(Int, responsedict["selected_response_number"]) # convert string "5" into integer 5
     catch
       errornote = "In your previous attempt, Selected_response_number was not a number. It must be a number."
       println("\nERROR SQLLLM compareState() Attempt $attempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
     end
 
-    println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
-    pprintln(Dict(responsedict))
+    # println("\n~~~ compareState() ")
+    # pprintln(Dict(responsedict))
+    # println("---\n", @__FILE__, ":", @__LINE__)
 
-    return responsedict[:selected_response_number]
+    return responsedict["selected_response_number"]
   end
   error("compareState() failed to generate an evaluation, Response: \n$response\n<|End of error|>", @__FILE__, ":", @__LINE__, " $(Dates.now())")
 end

src/util.jl

@@ -2,7 +2,7 @@ module util
 
 export makekey
 
-makekey(key, indice) = Symbol("$(key)_$indice")
+makekey(key, indice) = "$(key)_$indice"
 
 
 

system_prompt_template.jl

@@ -1,53 +1,60 @@
 """
-Default system message template:
+# -------------------------------- Default system message template ------------------------------- #
 
-  <Your role>
-    - You are a helpful assistant
-  </Your role>
-  <Situation>
-    - Describe the current situation
-  </Situation>
-  <Your vision>
-    - state your vision of how the situation will evolve, what would you want the situation to evolve into
-  </Your vision>
-  <Your mission>
-    - state the goal
-  </Your mission>
-  <Your mission's objective includes>
-    - Break the goal into smaller steps
-  </Your mission's objective includes>
-  <Your responsibility includes>
-    - state the mini goals that fall under your responsibility
-  </Your responsibility includes>
-  <Your responsibility does NOT includes>
+<Your role>
+  - You are a helpful assistant
+</Your role>
+<Situation>
+  - Describe the current situation
+  Ex. The world use enormous energy from non-sustainable sources. This leads to climate change.
+</Situation>
+<Your vision>
+  - state your vision of how the situation will evolve, what would you want the situation to evolve into
+  Ex. To be the leading innovator in sustainable technology by 2030, transforming global energy systems.
+</Your vision>
+<Your mission>
+  - state the goal
+  Ex. Empowering communities through clean energy solutions to create a sustainable future.
+</Your mission>
+<Your mission's objective includes>
+  - Specific, measurable, and time-bound goals that directly support the mission.
+  Ex. Launch 50 solar-powered water purification systems in 3 regions by 2025.
+</Your mission's objective includes>
+<Your responsibility includes>
+  - state the mini goals that fall under your responsibility
+</Your responsibility includes>
+<Your responsibility does NOT includes>
   - 
-  </Your responsibility does NOT includes>
-  <At each round of conversation, you will be given the following information>
+</Your responsibility does NOT includes>
+<At each round of conversation, you will be given the following information>
+  -
+</At each round of conversation, you will be given the following information>
+<You must follow the following policy>
+  -
+</You must follow the following policy>
+<You should follow the following guidelines>
+  -
+</You should follow the following guidelines>
+<You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+  Comprehension: State your comprehension about the current situation.
+  Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
+  Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names:
+    - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
+    - CHECKRESOURCES which you can use to check resources
+    - IMPLEMENT which you can use to implement the solution
+  Action_input: Detail the input for the action.
+</You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
+<You should only respond in format as described below>
+  Comprehension: ...
+  Plan: ...
+  Action_name: ...
+  Action_input: ...
+</You should only respond in format as described below>
+<Here are some examples>
 
-  </At each round of conversation, you will be given the following information>
-  <You must follow the following guidelines>
-    -
-  </You must follow the following guidelines>
-  <You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
-    Comprehension: State your comprehension about the current situation.
-    Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
-    Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names:
-      - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.
-      - CHECKRESOURCES which you can use to check resources
-      - IMPLEMENT which you can use to implement the solution
-    Action_input: Detail the input for the action.
-  </You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input>
-  <You should only respond in format as described below>
-    Comprehension: ...
-    Plan: ...
-    Action_name: ...
-    Action_input: ...
-  </You should only respond in format as described below>
-  <Here are some examples>
+</Here are some examples>
 
-  </Here are some examples>
-
-  Let's begin!
+Let's begin!
 
 
 
@@ -57,7 +64,7 @@ Default system message template:
 
 
 
-Example:
+# ------------------------------------------- Example: ------------------------------------------- #
 
 <Your profile>
 - You are a founder of a tech startup

test/runtests.jl

@@ -1,9 +1,9 @@
 using Revise
-using LibPQ, Dates, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
+using LibPQ, Dates, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
 using GeneralUtils, SQLLLM
 
 
-config = copy(JSON3.read("/appfolder/mountvolume/appdata/config.json"))
+config = JSON.parse("/appfolder/app/dev/YiemAgent/test/config.json")
 
 function executeSQL(sql::T) where {T<:AbstractString}
   host = config[:externalservice][:wineDB][:host]
@@ -29,13 +29,19 @@ function executeSQLVectorDB(sql)
   return result
 end
 
-function text2textInstructLLM(prompt::String; maxattempt=3)
+function text2textInstructLLM(prompt::String; maxattempt::Integer=3, modelsize::String="medium",
+    senderId=GeneralUtils.uuid4snakecase(), timeout=180,
+    llmkwargs=Dict(
+      :num_ctx => 32768,
+      :temperature => 0.5,
+    )
+  )
   msgMeta = GeneralUtils.generate_msgMeta(
     config[:externalservice][:loadbalancer][:mqtttopic];
     msgPurpose="inference",
     senderName="yiemagent",
-    senderId=sessionId,
-    receiverName="text2textinstruct_small",
+    senderId=senderId,
+    receiverName="text2textinstruct_$modelsize",
     mqttBrokerAddress=config[:mqttServerInfo][:broker],
     mqttBrokerPort=config[:mqttServerInfo][:port],
   )
@@ -44,16 +50,13 @@ function text2textInstructLLM(prompt::String; maxattempt=3)
     :msgMeta => msgMeta,
     :payload => Dict(
       :text => prompt,
-      :kwargs => Dict(
-        :num_ctx => 16384,
-        :temperature => 0.2,
-      )
+      :kwargs => llmkwargs
     )
   )
 
   response = nothing
   for attempts in 1:maxattempt
-    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=300, maxattempt=2)
+    _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=timeout, maxattempt=maxattempt)
     payload = _response[:response]
     if _response[:success] && payload[:text] !== nothing
       response = _response[:response][:text]
@@ -76,7 +79,7 @@ function getEmbedding(text::T) where {T<:AbstractString}
     msgPurpose="embedding",
     senderName="yiemagent",
     senderId=sessionId,
-    receiverName="text2textinstruct_small",
+    receiverName="textembedding",
     mqttBrokerAddress=config[:mqttServerInfo][:broker],
     mqttBrokerPort=config[:mqttServerInfo][:port],
   )
@@ -87,7 +90,8 @@ function getEmbedding(text::T) where {T<:AbstractString}
       :text => [text] # must be a vector of string
     )
   )
-  response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
+
+  response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120, maxattempt=3)
   embedding = response[:response][:embeddings]
   return embedding
 end
@@ -108,7 +112,6 @@ function findSimilarTextFromVectorDB(text::T1, tablename::T2, embeddingColumnNam
   return df
 end
 
-
 function similarSQLVectorDB(query; maxdistance::Integer=100)
   tablename = "sqlllm_decision_repository"
   # get embedding of the query
@@ -131,11 +134,10 @@ function similarSQLVectorDB(query; maxdistance::Integer=100)
   end
 end
 
-
 function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString}
   tablename = "sqlllm_decision_repository"
   # get embedding of the query
-  # query = state[:thoughtHistory][:question]
+  # query = state[:action_history][:question]
   df = findSimilarTextFromVectorDB(query, tablename, 
                                   "function_input_embedding", executeSQLVectorDB)
   row, col = size(df)
@@ -155,18 +157,77 @@ function insertSQLVectorDB(query::T1, SQL::T2; maxdistance::Integer=3) where {T1
   end
 end
 
-sessionId = "555"
+
+function similarSommelierDecision(recentevents::T1; maxdistance::Integer=3
+  )::Union{AbstractDict, Nothing} where {T1<:AbstractString}
+  tablename = "sommelier_decision_repository"
+  # find similar
+  println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__)
+  df = findSimilarTextFromVectorDB(recentevents, tablename, 
+                                  "function_input_embedding", executeSQLVectorDB)
+  row, col = size(df)
+  distance = row == 0 ? Inf : df[1, :distance]
+  if row != 0 && distance < maxdistance
+    # if there is usable decision, return it.
+    rowid = df[1, :id]
+    println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
+    output_b64 = df[1, :function_output_base64]   # pick the closest match
+    _output_str = String(base64decode(output_b64))
+    output = copy(JSON.parse(_output_str))
+    return output
+  else
+    println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
+    return nothing
+  end
+end
+
+
+function insertSommelierDecision(recentevents::T1, decision::T2; maxdistance::Integer=5
+  ) where {T1<:AbstractString, T2<:AbstractDict}
+  tablename = "sommelier_decision_repository"
+  # find similar
+  df = findSimilarTextFromVectorDB(recentevents, tablename, 
+                                  "function_input_embedding", executeSQLVectorDB)
+  row, col = size(df)
+  distance = row == 0 ? Inf : df[1, :distance]
+  if row == 0 || distance > maxdistance # no close enough SQL stored in the database
+    recentevents_embedding = getEmbedding(recentevents)[1]
+    recentevents = replace(recentevents, "'" => "")
+    decision_json = JSON.json(decision)
+    decision_base64 = base64encode(decision_json)
+    decision = replace(decision_json, "'" => "")
+    
+    sql = """
+          INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding');
+          """
+    println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__)
+    println(sql)
+    _ = executeSQLVectorDB(sql)
+  else
+    println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__)
+  end
+end
+
+sessionId = GeneralUtils.uuid4snakecase()
+d = Dict(:id => sessionId)
+filepath = "/appfolder/app/sessionid.json"
+open(filepath, "w") do io
+  JSON.pretty(io, d)
+end
+
+
 
 
 # query = "How many German wines do you have?"
-# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
+# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
 # selectedState = SQLLLM.compareState(query, highValueStateList, text2textInstructLLM)
 
 
 
 # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?")
-query = "How many French wines from Yiem store under 100 dollars do you have?"
+# query = "How many French wines from Yiem store under 100 dollars do you have?"
 # query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40"
+query = "from Yiem retailer, red wine from France. price 100 to 1000 USD. sweetness: 1-2, intensity: 4-5"
 # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza"
 # query = "wine_type: white, country: Austria, food to be served with wine: pork"
 # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"
@@ -291,8 +352,8 @@ SELECT * FROM wine WHERE wine_type = 'red' AND country = 'France' AND sweetness
 #   :evaluation =>
 #       "The user's question is to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. The thought is correct in identifying the conditions needed to filter the wine table. The action taken is to execute a SQL query to retrieve the desired data, which is also correct. The observation provides a search summary and two search results that match the user's question. Each result includes details about the wine such as ID, name, brand, manufacturer, region, country, type, grape variety, serving temperature, intensity, sweetness, tannin, and acidity.",
 #   :accepted_as_answer => "Yes",
-#   :thoughtHistory =>
-#       OrderedDict{Symbol, Any}(:question => "Search the database for wine_type: white, country: France, sweetness: 1", :thought_1 => "The user wants to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. To achieve this, we need to filter the wine table based on these conditions.", :action_name_1 => "GETDATA", :action_input_1 => "SELECT * FROM wine WHERE wine.wine_type = 'white' AND wine.country = 'France' AND wine.sweetness = 1;", :observation_1 => "\"Search summary: The resulting table represents wines.\\nSearch result: 1) wine_id: 5b6b6df9-d87c-4f33-8995-7249c2ecc917, wine_name: corton-charlemagne grand cru, brand: domaine des croix, manufacturer: domaine des croix, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n2) wine_id: 1ad27d16-ef64-4907-acf1-40631630c143, wine_name: puligny-montrachet 1er cru 'les demoiselles', brand: amiot guy, manufacturer: amiot guy, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n\\n\""),
+#   :action_history =>
+#       OrderedDict{String, Any}("question" => "Search the database for wine_type: white, country: France, sweetness: 1", "thought_1" => "The user wants to search the database for wines that have a type of \"white\", are from \"France\", and have a sweetness level of 1. To achieve this, we need to filter the wine table based on these conditions.", "action_name_1" => "GETDATA", "action_input_1" => "SELECT * FROM wine WHERE wine.wine_type = 'white' AND wine.country = 'France' AND wine.sweetness = 1;", "observation_1" => "\"Search summary: The resulting table represents wines.\\nSearch result: 1) wine_id: 5b6b6df9-d87c-4f33-8995-7249c2ecc917, wine_name: corton-charlemagne grand cru, brand: domaine des croix, manufacturer: domaine des croix, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n2) wine_id: 1ad27d16-ef64-4907-acf1-40631630c143, wine_name: puligny-montrachet 1er cru 'les demoiselles', brand: amiot guy, manufacturer: amiot guy, region: bourgogne, country: France, wine_type: white, grape_variety: cote de beaune blanc, serving_temperature: 11 to 13 Celsius, intensity: 4, sweetness: 1, tannin: missing, acidity: 3, fizziness: missing\\n\\n\""),
 #   :evaluationscore => 9,
 #   :select => nothing,
 #   :suggestion => "None")

test/test_1.jl

@@ -1,5 +1,5 @@
 using Revise
-using LibPQ, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
+using LibPQ, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
 using SQLLLM, GeneralUtils
 
 
@@ -39,7 +39,7 @@ df = DataFrame(result)
 
 
 
-config = copy(JSON3.read("config.json"))
+config = copy(JSON.parse("config.json"))
 
 msgMeta = GeneralUtils.generate_msgMeta(
     config[:externalservice][:text2textinstruct][:mqtttopic];

test/test_2.jl

@@ -1,8 +1,8 @@
 # using Revise
-# using SQLLLM, LLMMCTS, DataStructures, JSON3
+# using SQLLLM, LLMMCTS, DataStructures, JSON
 
 # query = "How many German wines do you have?"
-# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
+# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
 # selectedState = SQLLLM.compareState(query, highValueStateList)