diff --git a/src/interface.jl b/src/interface.jl index 962096a..b411fb1 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -158,6 +158,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query". - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. + - Do not use backticks (`). Use double quotes instead. You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action: 1) Understanding: @@ -183,39 +184,45 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, Let's begin! """ - + # [WORKING] add no backtick workprogress = "" for (k, v) in state[:thoughtHistory] if k ∉ [:query] workprogress *= "$k: $v\n" end end - - usermsg = - """ - $(context[:tablelist]) - User query: $(state[:thoughtHistory][:question]) - Hints: $similarSQL - Your Q&A: $QandA - Your work progress: $workprogress - Evaluation: $(state[:evaluation]) - Suggestion: $(state[:suggestion]) - """ - - _prompt = - [ - Dict(:name=> "system", :text=> systemmsg), - Dict(:name=> "user", :text=> usermsg) - ] - - # put in model format - prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct") - prompt *= - """ - <|start_header_id|>assistant<|end_header_id|> - """ + response = nothing # store for show when error msg show up + errornote = "" + noise = "" + for attempt in 1:10 + usermsg = + """ + $(context[:tablelist]) + User query: $(state[:thoughtHistory][:question]) + Hints: $similarSQL + Your Q&A: $QandA + Your work progress: $workprogress + Evaluation: $(state[:evaluation]) + Suggestion: $(state[:suggestion]) + $errornote + $noise + """ + + _prompt = + [ + Dict(:name=> "system", :text=> systemmsg), + Dict(:name=> "user", :text=> usermsg) + ] + + # put in model format + prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct") + prompt *= + """ + <|start_header_id|>assistant<|end_header_id|> + """ + try response = text2textInstructLLM(prompt) @@ -226,6 +233,10 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, delete!(responsedict, :observation) + if occursin('`', response) + response = replace(response, '`'=>"") + end + toollist = ["TABLEINFO", "GETDATA"] if responsedict[:action_name] ∉ toollist error("SQL decisionMaker() didn't use the given functions ", @__FILE__, " ", @__LINE__) @@ -258,220 +269,175 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, errorMsg = String(take!(io)) st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) println("") - println("Attempt $attempt. Error occurred: $errorMsg\n$st") + println("\n~~~ SQLLLM decisionMaker() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, " ", @__LINE__) println("") + noise = GeneralUtils.randstrings(3, 5) end end error("DecisionMaker failed to generate a thought ", response) end +# function decisionMaker(state::T1, context, text2textInstructLLM::Function, +# QandA::T2; similarSQL::Union{T3, Nothing}=nothing +# )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString} + +# similarSQL = +# if similarSQL === nothing +# "None" +# else +# "This is the closest matching SQL statement for a similar query: $similarSQL" +# end -# function decisionMaker(state::T2, config::T1 -# )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:AbstractDict} +# # lessonDict = +# # if isfile("lesson.json") +# # lessonDict = copy(JSON3.read("lesson.json")) +# # else +# # lessonDict = nothing +# # end -# if isfile("lesson.json") -# lessonDict = copy(JSON3.read("lesson.json")) -# else -# lessonDict = nothing -# end +# # lessonDict = nothing -# lesson = -# if lessonDict === nothing -# "" -# else -# """ -# You have attempted to help the user before and failed, either because your reasoning for the -# recommendation was incorrect or your response did not exactly match the user expectation. -# The following lesson(s) give a plan to avoid failing to help the user in the same way you -# did previously. Use them to improve your strategy to help the user. +# # lesson = +# # if lessonDict === nothing +# # "" +# # else +# # """ +# # You have attempted to help the user before and failed, either because your reasoning for the +# # recommendation was incorrect or your response did not exactly match the user expectation. +# # The following lesson(s) give a plan to avoid failing to help the user in the same way you +# # did previously. Use them to improve your strategy to help the user. -# Here are some lessons in JSON format: -# $(JSON3.write(lessonDict)) +# # Here are some lessons in JSON format: +# # $(JSON3.write(lessonDict)) -# When providing the thought and action for the current trial, that into account these failed -# trajectories and make sure not to repeat the same mistakes and incorrect answers. -# """ -# end +# # When providing the thought and action for the current trial, that into account these failed +# # trajectories and make sure not to repeat the same mistakes and incorrect answers. +# # """ +# # end -# _prompt = +# systemmsg = # """ -# You are a helpful data engineer. -# Your goal is to help the user to get what the user wants. -# You are also keen to improve your helpfulness with lesson(s). - -# You must follow the following criteria: -# 1) Get to know what table are available in the database. -# 2) Get to know what the data in the table looks like. -# 3) If you can't find a single table that can be used to answer the user's question, try joining multiple tables to see if you can obtain the answer. -# 4) If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your question". - -# You should only respond with interleaving Thought, Action, Observation steps. -# Thought can reason about the current situation, and Action can be one of the following functions: -# 1) listalltables[NA], which you can use to list all tables in the database and see their descriptions. "NA" word is the function input. -# 2) tableinfo[table_name], which you can use to see the table and its column description. "table_name" is name of the table you want to get info. -# 3) getdata[instructions], which you can use to ask other people to get the data from tables for you. "instructions" should clearly describe how you want others to extract the data. -# For example, -# a. Query the "Engine" table to identify the engine types that have 3 cylinders. This can be done using a SELECT statement in SQL, filtering the results where the number of cylinders equals 3. -# b. Once you have identified the engine types with 3 cylinders, use this information to query the "Car" table. You're looking for car models that are associated with these engine types. This can be achieved by performing a JOIN operation between the "Car" and "Engine" tables based on the engine type. -# 4) finalanswerbox[answer], which returns your answer to the user. "answer" is your answer to the user question. -# After each observation, provide the next Thought and next Action. - -# You should only respond in JSON format as describe below: -# { -# "thought": "your reasoning", -# "action": {"name": "action to take", "input": "Action input"}, -# "observation": "result of the action" -# } - -# Here are some examples: -# { -# "question": "I would like to buy a sedan with 8 seats.", -# "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.", -# "action_1": {"name": "inventory", "input": "sedan with 8 seats."}, -# "observation_1": "Several model has 8 seats. Available color are black, red green" -# } -# { -# "thought": "I have a few color for the user to choose from. I will ask him what color he likes.", -# "action": {"name": "chatbox", "input": "Which color do you like?"} -# "observation": "I'll take black." -# } +# You are a helpful assistant that get the data from a database to satisfy the user's query. +# You are also eager to improve your helpfulness. -# $lesson +# At each round of conversation, the user will give you the current situation: +# User Query: ... +# Hints: ... +# Your Q&A: ... +# Your work progress: ... +# Evaluation: Evaluation of the latest action and observation +# Suggestion: ... + +# You should consider the following guidelines: +# - Do not create any table in the database +# - Column name can be the same in different tables. Refer to column comments to get more details by using TABLEINFO function +# - A junction table can be used to link tables together. Another use case is for filtering data. +# - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. +# - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query". +# - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. + +# You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action: +# 1) Understanding: +# - State your understanding about the current situation. +# 2) Reasoning: +# - State your step by step reasoning about the current situation. +# 3) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. +# 4) Action_name (Must be aligned with your plan): Can be one of the following functions: +# - TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. "list_of_table_name" is a list of table name you want to get info. e.g. TABLEINFO["table name 1", "table name 2"] +# - GETDATA[SQL], which you can use to get the data from the database. "SQL" is the single SQL command to be executed against the database. +# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. +# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. +# 5) Action_input: Input to the action +# 6) Observation: Result of the immediately preceding action + +# You should only respond in format as described below: +# Understanding: ... +# Reasoning: ... +# Plan: ... +# Action_name: ... +# Action_input: ... +# Observation: ... # Let's begin! - -# $(JSON3.write(state[:thoughtHistory])) -# {"thought" # """ - -# # _prompt = -# # """ -# # You are a helpful data engineer. -# # Your goal is to help the user to get what the user wants. -# # You are also keen to improve your helpfulness with lesson(s). - -# # You must follow the following criteria: -# # 1) Get to know what table are available in the database. -# # 2) Get to know what the data in the table looks like. -# # 3) If you can't find a single table that can be used to answer the user's question, try joining multiple tables to see if you can obtain the answer. -# # 4) Keep trying even if you get SQL execution error. - -# # You should only respond with interleaving Thought, Action, Observation steps. -# # Thought can reason about the current situation, and Action can be one of the following functions: -# # 1) listalltables[NA], which you can use to list all tables in the database and see their descriptions. "NA" word is the function input. -# # 2) tableinfo[table_name], which you can use to see the table and its column description. "table_name" is name of the table you want to get info. -# # 3) getdata[SQL], which you can use to ask other people to get the data from tables for you. "SQL" is the command you will use to extract the data. -# # 4) finalanswerbox[answer], which returns your answer to the user. "answer" is your answer to the user question. -# # After each observation, provide the next Thought and next Action. - -# # You should only respond in JSON format as describe below: -# # { -# # "thought": "your reasoning", -# # "action": {"name": "action to take", "input": "Action input"}, -# # "observation": "result of the action" -# # } - -# # Here are some examples: -# # { -# # "question": "I would like to buy a sedan with 8 seats.", -# # "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.", -# # "action_1": {"name": "inventory", "input": "sedan with 8 seats."}, -# # "observation_1": "Several model has 8 seats. Available color are black, red green" -# # } -# # { -# # "thought": "I have a few color for the user to choose from. I will ask him what color he likes.", -# # "action": {"name": "chatbox", "input": "Which color do you like?"} -# # "observation": "I'll take black." -# # } - -# # $lesson - -# # Let's begin! - -# # $(JSON3.write(state[:thoughtHistory])) -# # {"thought" -# # """ - -# # apply LLM specific instruct format -# externalService = config[:externalservice][:text2textinstruct] -# llminfo = externalService[:llminfo] -# prompt = -# if llminfo[:name] == "llama3instruct" -# GeneralUtils.formatLLMtext_llama3instruct("system", _prompt) -# else -# error("llm model name is not defied yet $(@__LINE__)") +# workprogress = "" +# for (k, v) in state[:thoughtHistory] +# if k ∉ [:query] +# workprogress *= "$k: $v\n" +# end # end -# msgMeta = GeneralUtils.generate_msgMeta( -# externalService[:mqtttopic], -# senderName= "decisionMaker", -# senderId= string(uuid4()), -# receiverName= "text2textinstruct", -# mqttBroker= config[:mqttServerInfo][:broker], -# mqttBrokerPort= config[:mqttServerInfo][:port], -# ) +# usermsg = +# """ +# $(context[:tablelist]) +# User query: $(state[:thoughtHistory][:question]) +# Hints: $similarSQL +# Your Q&A: $QandA +# Your work progress: $workprogress +# Evaluation: $(state[:evaluation]) +# Suggestion: $(state[:suggestion]) +# """ -# outgoingMsg = Dict( -# :msgMeta=> msgMeta, -# :payload=> Dict( -# :text=> prompt, -# :kwargs=> Dict( -# :max_tokens=> 512, -# :stop=> ["<|eot_id|>"], -# ) -# ) -# ) -# @show outgoingMsg +# _prompt = +# [ +# Dict(:name=> "system", :text=> systemmsg), +# Dict(:name=> "user", :text=> usermsg) +# ] -# for attempt in 1:5 +# # put in model format +# prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct") +# prompt *= +# """ +# <|start_header_id|>assistant<|end_header_id|> +# """ +# response = nothing # store for show when error msg show up +# for attempt in 1:10 # try -# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) -# _responseJsonStr = response[:response][:text] -# expectedJsonExample = -# """ -# Here is an expected JSON format: -# { -# "thought": "...", -# "action": {"name": "...", "input": "..."}, -# "observation": "..." -# } -# """ -# responseJsonStr = FormatCorrector.jsoncorrection(config, _responseJsonStr, expectedJsonExample) -# thoughtDict = copy(JSON3.read(responseJsonStr)) +# response = text2textInstructLLM(prompt) -# # check if dict has all required value -# thought::AbstractString = thoughtDict[:thought] -# actionname::AbstractString = thoughtDict[:action][:name] -# actioninput::AbstractString = thoughtDict[:action][:input] -# if actionname ∈ ["listalltables", "tableinfo", "getdata", "finalanswerbox"] -# # LLM use available function -# elseif thought == "" -# error("DecisionMaker has no thought") -# elseif length(actioninput) == 0 -# error("DecisionMaker has no actioninput") -# else -# error("DecisionMaker use wrong function") +# # textToDict() search for action_input +# responsedict = GeneralUtils.textToDict(response, +# ["Understanding", "Reasoning", "Plan", "Action_name", "Action_input", "Observation"], +# rightmarker=":", symbolkey=true, lowercasekey=true) + +# delete!(responsedict, :observation) + +# toollist = ["TABLEINFO", "GETDATA"] +# if responsedict[:action_name] ∉ toollist +# error("SQL decisionMaker() didn't use the given functions ", @__FILE__, " ", @__LINE__) +# end + +# for i in toollist +# if occursin(i, responsedict[:action_input]) +# error("Action_name is in action_input which is not allowed.") +# end +# end + +# for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input] +# if length(JSON3.write(responsedict[i])) == 0 +# error("$i is empty ", @__FILE__, " ", @__LINE__) +# end # end # # check if there are more than 1 key per categories -# for i ∈ ["thought", "action", "observation"] -# matchkeys = GeneralUtils.findMatchingDictKey(thoughtDict, i) +# for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input] +# matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i) # if length(matchkeys) > 1 # error("DecisionMaker has more than one key per categories") # end # end -# return thoughtDict +# return responsedict # catch e # io = IOBuffer() # showerror(io, e) # errorMsg = String(take!(io)) # st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) # println("") -# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st" +# println("Attempt $attempt. Error occurred: $errorMsg\n$st") # println("") # end # end -# error("DecisionMaker failed to generate a thought") +# error("DecisionMaker failed to generate a thought ", response) # end @@ -1221,6 +1187,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; - State your understanding about the current situation. 2) Q: Given the situation, "ask yourself" about the situation at least five, but no more than ten, questions. 3) A: Given the situation, "answer to yourself" the best you can. + - Do not generate any text after the last answer. You must only respond in format as described below: Understanding: ... @@ -1244,9 +1211,9 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; response = nothing # store for show when error msg show up errornote = "" + noise = "" for attempt in 1:10 - usermsg = """ $(context[:tablelist]) @@ -1254,6 +1221,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; Hints: $similarSQL Your work progress: $workprogress $errornote + $noise """ _prompt = @@ -1271,10 +1239,18 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; try response = text2textInstructLLM(prompt) + + # check if response is valid q_number = count("Q", response) if q_number < 1 - error("too few questions only $q_number questions are generated ", @__FILE__, " ", @__LINE__) + errornote = "too few question" + error("too few questions only $q_number questions are generated") end + if occursin('`', response) + response = replace(response, '`'=>"") + end + + # response = string(split(response, "Please")[1]) # LLM usually add comments which is no need. responsedict = GeneralUtils.textToDict(response, ["Understanding", "Q1"], @@ -1288,9 +1264,8 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; showerror(io, e) errorMsg = String(take!(io)) st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) - println("") - println("Attempt $attempt. Error occurred: $errorMsg\n$st") - println("") + println("\n~~~ SQLLLM generatequestion() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, " ", @__LINE__) + noise = GeneralUtils.randstrings(3, 5) end end error("generatequestion failed to generate a thought ", response) diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 7a158a9..87146c6 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -653,7 +653,7 @@ function SQLexecution(executeSQL::Function, sql::T tablesize = size(df) row, column = tablesize if row == 0 # if 0 row - error("The resulting table has 0 row. Possible causes: 1) You might be searching in the wrong place 2) There could be a typo in your search query 3) No data matches your search criteria.") + error("The resulting table has 0 row. Possible causes: 1) You might be searching in the wrong place 2) There could be a typo in your search query.") elseif column > 30 error("SQL execution failed. An unexpected error occurred. Please try again.") end