Merge pull request 'WIP v0.2.2-dev' (#1) from v0.2.2-dev into main

Reviewed-on: #1

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.11.2"
 manifest_format = "2.0"
-project_hash = "7276bacf2126ccf319f58f1cc1aab4f9a73ac360"
+project_hash = "6e88822413ea4a623cd914d84de127dc6c57fceb"
 
 [[deps.AliasTables]]
 deps = ["PtrArrays", "Random"]
@@ -306,11 +306,11 @@ version = "1.19.3+0"
 
 [[deps.LLMMCTS]]
 deps = ["GeneralUtils", "JSON3"]
-git-tree-sha1 = "5be792515b65e464d15ac8db6a9f6f6628c7d81a"
+git-tree-sha1 = "c8ad9715e78bbd19f5ac79e1f1cacf85f141449d"
 repo-rev = "main"
 repo-url = "https://git.yiem.cc/ton/LLMMCTS"
 uuid = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
-version = "0.1.0"
+version = "0.1.2"
 
 [[deps.LaTeXStrings]]
 git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"

Project.toml

@@ -1,7 +1,7 @@
 name = "SQLLLM"
 uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
 authors = ["narawat lamaiin <narawat@outlook.com>"]
-version = "0.1.0"
+version = "0.2.2"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -23,5 +23,5 @@ URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
-GeneralUtils = "0.1.0"
+GeneralUtils = "0.1, 0.2"
-LLMMCTS = "0.1.0"
+LLMMCTS = "0.1"

src/interface.jl

@@ -133,25 +133,26 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
 
   systemmsg = 
   """
-    You are a helpful assistant that get the data from a database to satisfy the user's query. 
+    You are a helpful assistant that find the data from a database to satisfy the user's query. 
     You are also eager to improve your helpfulness.
 
     At each round of conversation, the user will give you the current situation:
     User Query: ...
-    Hints: ...
+    Example: ...
     Your Q&A: ...
     Your work progress: ...
     Evaluation: Evaluation of the latest action and observation
     Suggestion: ...
     
-    You should consider the following guidelines:
+    You must follow the following guidelines:
+    - Keep SQL queries focused only on the provided information.
+
+    You should follow the following guidelines:
     - Do not create any table in the database
-    - Column name can be the same in different tables. Refer to column comments to get more details by using TABLEINFO function
     - A junction table can be used to link tables together. Another use case is for filtering data.
     - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
     - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
     - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
-    - Do not use backticks (`). Use double quotes instead.
 
     You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action:
     1) Understanding: 
@@ -160,8 +161,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
         - State your step by step reasoning about the current situation.
     3) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
     4) Action_name (Must be aligned with your plan): Can be one of the following functions: 
-        - TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. "list_of_table_name" is a list of table name you want to get info. e.g. TABLEINFO["table name 1", "table name 2"]
+        - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database.
-        - GETDATA[SQL], which you can use to get the data from the database. "SQL" is the single SQL command to be executed against the database.
           For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
           Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
     5) Action_input: Input to the action
@@ -173,7 +173,6 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
     Plan: ...
     Action_name: ...
     Action_input: ...
-    Observation: ...
 
     Let's begin!
   """
@@ -203,7 +202,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
     """
     $(context[:tablelist])
     User query: $(state[:thoughtHistory][:question])
-    Hints: $similarSQL_
+    Example: $similarSQL_
     Your Q&A: $QandA
     Your work progress: $workprogress
     Evaluation: $(state[:evaluation])
@@ -223,60 +222,87 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
       """
       <|start_header_id|>assistant<|end_header_id|>
       """
+    response = text2textInstructLLM(prompt)
+    println("\nSQL decisionMaker() rawresponse: \n", response)
 
-    try
+    if occursin("NULL", response)
-      response = text2textInstructLLM(prompt)
+      errornote = "\nSQL decisionMaker() NULL response is not allowed"
-
+      println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
-      # textToDict() search for action_input
+      continue
-      responsedict = GeneralUtils.textToDict(response,
-                  ["Understanding", "Reasoning", "Plan", "Action_name", "Action_input", "Observation"], 
-                  rightmarker=":", symbolkey=true, lowercasekey=true)
-
-      delete!(responsedict, :observation)
-
-      # remove backticks
-      if occursin("```", responsedict[:action_input])
-        responsedict[:action_input] = 
-          GeneralUtils.extract_triple_backtick_text(responsedict[:action_input])
-      end
-
-      toollist = ["TABLEINFO", "GETDATA"]
-      if responsedict[:action_name] ∉ toollist
-        error("SQL decisionMaker() didn't use the given functions ", @__FILE__, " ", @__LINE__)
-      end
-
-      for i in toollist
-        if occursin(i, responsedict[:action_input])
-          error("Action_name is in action_input which is not allowed.")
-        end
-      end
-
-      for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
-        if length(JSON3.write(responsedict[i])) == 0
-          error("$i is empty ", @__FILE__, " ", @__LINE__)
-        end
-      end
-
-      # check if there are more than 1 key per categories
-      for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
-        matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
-        if length(matchkeys) > 1
-          error("DecisionMaker has more than one key per categories")
-        end
-      end
-
-      state[:decisionMaker] = responsedict
-
-      return responsedict
-    catch e
-      io = IOBuffer()
-      showerror(io, e)
-      errorMsg = String(take!(io))
-      st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
-      println("")
-      println("\n~~~ SQLLLM decisionMaker() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, " ", @__LINE__)
-      println("")
     end
+
+    header = ["Understanding", "Reasoning", "Plan", "Action_name", "Action_input", "Observation"]
+
+    # detect if there are more than 1 key per categories
+    count = GeneralUtils.countGivenWords(response, header)
+    duplicateKeywordFlag = false
+    for (i, v) in enumerate(count)
+      keyword = header[i]
+      keywordNumber = v
+      if keywordNumber > 1
+        errornote = "\nSQL query has duplicated keyword, $keyword"
+        println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
+        duplicateKeywordFlag = true
+        break
+      end
+    end
+    duplicateKeywordFlag == true ? continue : nothing
+
+    # textToDict() search for action_input
+    responsedict = GeneralUtils.textToDict(response, header, 
+                rightmarker=":", symbolkey=true, lowercasekey=true)
+
+    delete!(responsedict, :observation)
+
+    # remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String})
+    if occursin("```", responsedict[:action_input])
+      sql = GeneralUtils.extract_triple_backtick_text(responsedict[:action_input])[1]
+      if sql[1:4] == "sql\n"
+        sql = sql[5:end]
+      end
+      sql = split(sql, ';')   # some time there are comments in the sql
+      sql = sql[1] * ';'
+
+      responsedict[:action_input] = sql
+    end
+
+    toollist = ["TABLEINFO", "GETDATA"]
+    if responsedict[:action_name] ∉ toollist
+      errornote = "\nYou must only use the given functions"
+      println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
+      continue
+    end
+
+    for i in toollist
+      if occursin(i, responsedict[:action_input])
+        errornote = "\n action_name is in action_input which is not allowed."
+        println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
+        continue
+      end
+    end
+
+    for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
+      if length(JSON3.write(responsedict[i])) == 0
+        errornote = "\n $i is empty"
+        println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
+        continue
+      end
+    end
+
+    # check if there are more than 1 key per categories
+    for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
+      matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
+      if length(matchkeys) > 1
+        errornote = "\n $i has more than one key"
+        println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
+        continue
+      end
+    end
+
+    state[:decisionMaker] = responsedict
+
+    return responsedict
+
   end
   error("DecisionMaker failed to generate a thought ", response)
 end
@@ -302,7 +328,7 @@ julia>
 # Signature
 """
 function evaluator(state::T1, text2textInstructLLM::Function;
-  addSQLVectorDB::Union{Function, Nothing}=nothing
+  insertSQLVectorDB::Union{Function, Nothing}=nothing
   ) where {T1<:AbstractDict}
   
   # systemmsg = 
@@ -446,8 +472,9 @@ function evaluator(state::T1, text2textInstructLLM::Function;
   "reasoning" is agent's step-by-step reasoning about the current situation
   "plan" is agent's plan to complete the task from the current situation
   "action_name" is the name of the action taken, which can be one of the following functions:
-      - TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. "list_of_table_name" is a list of table name you want to get info. e.g. TABLEINFO["table name 1", "table name 2"]
+      - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database.
-      - GETDATA[SQL], which you can use to get the data from the database. "SQL" is the single SQL command to be executed against the database.
+          For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
+          Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
   "action_input" is the input to the action
   "observation" is result of the preceding immediate action
 
@@ -768,7 +795,7 @@ function transition(state::T, args::NamedTuple
   context = args[:context]
   executeSQL::Function = args[:executeSQL]
   text2textInstructLLM::Function = args[:text2textInstructLLM]
-  addSQLVectorDBF::Function = args[:addSQLVectorDB]
+  insertSQLVectorDB::Function = args[:insertSQLVectorDB]
   querySQLVectorDBF::Function = args[:querySQLVectorDB]
 
   # getting SQL from vectorDB
@@ -781,7 +808,7 @@ function transition(state::T, args::NamedTuple
       # so that other simulation start from this same node is not contaminated with actioninput
       listAllTable_json(executeSQL)
     elseif thoughtDict[:action_name] == "TABLEINFO"
-      input = copy(JSON3.read(thoughtDict[:action_input]))
+      input = thoughtDict[:action_input]  # BUG thoughtDict[:action_input] = "\"wine\""
       tableinfo(executeSQL, input)
     elseif thoughtDict[:action_name] == "GETDATA"
       response = SQLexecution(executeSQL, thoughtDict[:action_input])
@@ -804,7 +831,7 @@ function transition(state::T, args::NamedTuple
   isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false
   newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON3.write(result), select, reward, isterminal)
   progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM; 
-                                      addSQLVectorDB=addSQLVectorDBF)
+                                    insertSQLVectorDB=insertSQLVectorDB)
 
   return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue)
 end
@@ -887,7 +914,7 @@ julia> println(result)
 # Signature
 """
 function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
-  addSQLVectorDB::Union{Function, Nothing}=nothing,
+  insertSQLVectorDB::Union{Function, Nothing}=nothing,
   similarSQLVectorDB::Union{Function, Nothing}=nothing,
   ) where {T<:AbstractString}
 
@@ -931,7 +958,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
                     executeSQL=executeSQL, 
                     text2textInstructLLM=text2textInstructLLM,
                     querySQLVectorDB=similarSQLVectorDB,
-                    addSQLVectorDB=addSQLVectorDB,
+                    insertSQLVectorDB=insertSQLVectorDB,
                   )
 
   earlystop(state) = state[:reward] >= 8 ? true : false
@@ -945,8 +972,14 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
   extracted = resultState[:thoughtHistory][latestKey]
 
   # add to vectorDB only if the answer is achieved and the state is terminal 
-  if addSQLVectorDB !== nothing && resultState[:isterminal] == true
+  if insertSQLVectorDB !== nothing && resultState[:isterminal] == true &&
-    addSQLVectorDB(resultState[:thoughtHistory][:question], sql)
+    resultState[:rawresponse] !== nothing
+
+    insertSQLVectorDB(resultState[:thoughtHistory][:question], sql)
+  end
+
+  if extracted === nothing  #BUG
+    println("query() return nothing")
   end
 
   return (text=extracted, rawresponse=resultState[:rawresponse]) 
@@ -1017,11 +1050,11 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
 
   systemmsg = 
   """
-    You are a helpful assistant that generate multiple questions about the current situation.
+    You are a SQL expert that generate multiple questions about the current situation.
     
     At each round of conversation, the user will give you the current situation:
     User query: ...
-    Hints: ...
+    Example: ...
     Your work progress: ...
 
     About the tables in the database:
@@ -1035,6 +1068,9 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
     3) Some information can be accessed by joining multiple tables.
     4) Do not generate any question or comments at the end.
 
+    You should follow the following guidelines:
+    - When querying data in the database, start with broad search terms and refine your query later for more precise results. 
+
     You should then respond to the user with:
     1) Understanding: 
         - State your understanding about the current situation.
@@ -1052,6 +1088,13 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
     A3: ...
     ...
 
+    Here are some examples:
+    Q: What information in the hints is not necessary based on the query?
+    A: Country is not specified in the query thus it should not be included in an SQL
+
+    Q: How can I modify a SQL example to fit my specific query needs?
+    A: ...
+
     Let's begin!
   """
   
@@ -1071,7 +1114,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
     """
     $(context[:tablelist])
     User query: $(state[:thoughtHistory][:question])
-    Hints: $similarSQL
+    Example: $similarSQL
     Your work progress: $workprogress
     $errornote
     $noise

src/llmfunction.jl

@@ -520,65 +520,6 @@ julia> response = SQLLLM.SQLexecution(executeSQL, sql)
 
 # Signature
 """
-# function SQLexecution(executeSQL::Function, sql::T
-#   )::NamedTuple{(:result, :success, :errormsg, :reward, :isterminal), Tuple{Union{DataFrame, Nothing}, Bool, Union{String, Nothing}, Integer, Bool}} where {T<:AbstractString}
-#   println("\n~~~ 1-01 ", @__FILE__, " ", @__LINE__) 
-#   #XXX dummy SQL. use for testing
-#   # sql = "SELECT w.wine_name FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.\"food_name\" = 'lamb';"
-#   # sql = " SELECT      w.wine_name FROM      wine w JOIN      food f ON f.food_name = 'lamb' JOIN      wine_food wf ON w.wine_id = wf.wine_id AND f.food_id = wf.food_id GROUP BY      w.wine_name ORDER BY      COUNT(DISTINCT w.wine_id) DESC;"
-#   # sql = " SELECT COUNT(DISTINCT wf.wine_id) FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.food_name ILIKE '%lamb%'"
-
-#   #XXX use for package testing, remove when done
-#   # ans = "1.schilfwein zweigelt 2.cabernet sauvignon reserve limited edition"
-#   # ans = "There are 1500 wines that can be paired with lamb."
-#   # ans = "1500"
-#   # return (response=ans, errormsg=nothing, reward=1, isterminal=true)
-
-#   # add LIMIT to the SQL to prevent loading large data
-#   sql = strip(sql)
-#   println("\n~~~ SQL 1", @__FILE__, " ", @__LINE__)
-#   println(sql)
-#   println("\n~~~ 1-02 ", @__FILE__, " ", @__LINE__) 
-
-#   if sql[end] != ';'
-#     errorMsg = "Error, SQL execution failed because it does not ended with ';'"
-#     return (result=nothing, success=false, errormsg=errorMsg, reward=0, isterminal=false)
-#   end
-#   println("\n~~~ 1-03 ", @__FILE__, " ", @__LINE__) 
-#   if !occursin("LIMIT", sql)
-#     # sql = sql[1:end-1] * " LIMIT 100;"
-#     sql = sql[1:end-1] * " ORDER BY RANDOM() LIMIT 2;"
-#   end
-
-#   println("\n~~~ SQL 2", @__FILE__, " ", @__LINE__)
-#   println(sql)
-#   println("\n~~~ 1-1 ", @__FILE__, " ", @__LINE__) 
-#   result = executeSQL(sql)
-#   println("\n~~~ 1-2 ", @__FILE__, " ", @__LINE__) 
-#   df = DataFrame(result)
-#   println("\n~~~ raw df ", df)
-#   tablesize = size(df)
-#   println("\n~~~ df size ", tablesize)
-#   println("\n~~~ 6 ", @__FILE__, " ", @__LINE__) 
-#   row = tablesize[1]
-#   println("\n~~~ 7 ", @__FILE__, " ", @__LINE__) 
-#   if row == 0  # if 0 row
-#     errorMsg = "The resulting table has 0 row. Possible causes: 1) SQL is incorrect 2) There is no data that match your search criteria."
-#     return (result=nothing, success=false, errormsg=errorMsg, reward=0, isterminal=false)
-#   end
-#   println("\n~~~ 8 ", @__FILE__, " ", @__LINE__) 
-#   df1 =   
-#     if row > 2
-#       # ramdom row to pick
-#       df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df
-#     else
-#       df
-#     end
-
-#   println("\n~~~ SQLexecution result ", @__FILE__, " ", @__LINE__)   
-#   println(df1)
-#   return (result=df1, success=true, errormsg=nothing, reward=1, isterminal=true)
-# end
 function SQLexecution(executeSQL::Function, sql::T
 ) where {T<:AbstractString}
 
@@ -596,9 +537,12 @@ function SQLexecution(executeSQL::Function, sql::T
 
     # add LIMIT to the SQL to prevent loading large data
     sql = strip(sql)
+
+    # remove DISTINCT keyword because it is incompatible with RANDOM()
+    sql = replace(sql, "DISTINCT" => "")  
+
     if sql[end] == ';'
       if !occursin("LIMIT", sql)
-        # sql = sql[1:end-1] * " LIMIT 100;"
         sql = sql[1:end-1] * " ORDER BY RANDOM() LIMIT 2;"
       end
     else
@@ -613,7 +557,7 @@ function SQLexecution(executeSQL::Function, sql::T
     tablesize = size(df)
     row, column = tablesize
     if row == 0  # if 0 row
-      error("The resulting table has 0 row. Possible causes: 1) You might be searching in the wrong place 2) There could be a typo in your search query.")
+      error("The resulting table has 0 row. Possible causes: 1) Your search criteria might be too specific. Relaxing some conditions could yield better results. Remember, you can always refine your search later. 2) There could be a typo in your search query. 3) You might be searching in the wrong place.")
     elseif column > 30
       error("SQL execution failed. An unexpected error occurred. Please try again.")
     end