1346 lines
50 KiB
Julia
1346 lines
50 KiB
Julia
module interface
|
|
|
|
export decisionMaker, evaluator, reflector, transition, query
|
|
|
|
using LibPQ, DataStructures, JSON3, UUIDs, PrettyPrinting
|
|
using GeneralUtils, LLMMCTS
|
|
using ..util, ..llmfunction
|
|
|
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
|
|
|
|
|
|
|
""" Think and choose action.
|
|
|
|
# Arguments
|
|
- `state::T2`
|
|
A game state
|
|
- `context`
|
|
A context that will be added to decisionMaker
|
|
- `text2textInstructLLM::Function`
|
|
A function that handles communication to LLM service
|
|
|
|
# Return
|
|
- `thoughtDict::Dict{Symbol, Any}`
|
|
|
|
# Example
|
|
```jldoctest
|
|
julia> using SQLLLM, GeneralUtils, UUIDs, DataStructures, PrettyPrinting
|
|
julia> state = Dict(
|
|
:isterminal => false,
|
|
:lesson => nothing,
|
|
:reward => 0,
|
|
:evaluation => "None",
|
|
:accepted_as_answer => "No",
|
|
:thoughtHistory => OrderedDict{Symbol, Any}(:question => "How many wines do you have that can be paired with lamb?"),
|
|
:evaluationscore => 0,
|
|
:suggestion => "None"
|
|
)
|
|
julia> context = Dict(:tablelist=> "None")
|
|
julia> function text2textInstructLLM(prompt::String)
|
|
config = Dict(
|
|
:mqttServerInfo => Dict(
|
|
:description => "mqtt server info",
|
|
:port => 1883,
|
|
:broker => "mqtt.yiem.cc"
|
|
),
|
|
:externalservice => Dict(
|
|
:text2textinstruct => Dict(
|
|
:mqtttopic => "/loadbalancer/requestingservice",
|
|
:description => "text to text service with instruct LLM",
|
|
:llminfo => Dict(:name => "llama3instruct")
|
|
),
|
|
)
|
|
)
|
|
|
|
# apply LLM specific instruct format
|
|
externalService = config[:externalservice][:text2textinstruct]
|
|
|
|
msgMeta = GeneralUtils.generate_msgMeta(
|
|
externalService[:mqtttopic],
|
|
senderName= "SQLLLM",
|
|
senderId= string(uuid4()),
|
|
receiverName= "text2textinstruct",
|
|
mqttBroker= config[:mqttServerInfo][:broker],
|
|
mqttBrokerPort= config[:mqttServerInfo][:port],
|
|
)
|
|
|
|
outgoingMsg = Dict(
|
|
:msgMeta=> msgMeta,
|
|
:payload=> Dict(
|
|
:text=> prompt,
|
|
:kwargs=> Dict(
|
|
:max_tokens=> 512,
|
|
:stop=> ["<|eot_id|>"],
|
|
:temperature=> 0.2,
|
|
)
|
|
)
|
|
)
|
|
|
|
_response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
|
response = _response[:response][:text]
|
|
|
|
return response
|
|
end
|
|
julia> result = SQLLLM.decisionMaker(state, context, text2textInstructLLM)
|
|
julia> pprintln(result)
|
|
Dict(
|
|
:action_input => "[\"wine_food\"]",
|
|
:thought =>
|
|
"Since the user is asking about wine pairing, I need to find a way to connect the \"wine\" and \"food\" tables. The \"wine_food\" table seems like a good starting point.",
|
|
:plan =>
|
|
"First, I'll get information about the \"wine_food\" table to see how it relates to the other two tables. Then, I'll use this information to craft an instruction that retrieves the wines that can be paired with lamb.",
|
|
:observation => "[{\"name\": \"wine_food\", \"columns\": [\"wine_id\", \"food_id\"]}]",
|
|
:action_name => "TABLEINFO"
|
|
)
|
|
```
|
|
|
|
# TODO
|
|
- [] implement RAG to pull similar experience
|
|
|
|
# Signature
|
|
"""
|
|
function decisionMaker(state::T1, context, text2textInstructLLM::Function,
|
|
QandA::T2; similarSQL::Union{T3, Nothing}=nothing
|
|
)::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
|
|
|
|
similarSQL =
|
|
if similarSQL === nothing
|
|
"None"
|
|
else
|
|
"This is the closest matching SQL statement for a similar query: $similarSQL"
|
|
end
|
|
|
|
# lessonDict =
|
|
# if isfile("lesson.json")
|
|
# lessonDict = copy(JSON3.read("lesson.json"))
|
|
# else
|
|
# lessonDict = nothing
|
|
# end
|
|
|
|
# lessonDict = nothing
|
|
|
|
# lesson =
|
|
# if lessonDict === nothing
|
|
# ""
|
|
# else
|
|
# """
|
|
# You have attempted to help the user before and failed, either because your reasoning for the
|
|
# recommendation was incorrect or your response did not exactly match the user expectation.
|
|
# The following lesson(s) give a plan to avoid failing to help the user in the same way you
|
|
# did previously. Use them to improve your strategy to help the user.
|
|
|
|
# Here are some lessons in JSON format:
|
|
# $(JSON3.write(lessonDict))
|
|
|
|
# When providing the thought and action for the current trial, that into account these failed
|
|
# trajectories and make sure not to repeat the same mistakes and incorrect answers.
|
|
# """
|
|
# end
|
|
|
|
systemmsg =
|
|
"""
|
|
You are a helpful assistant that get the data from a database to satisfy the user's query.
|
|
You are also eager to improve your helpfulness.
|
|
|
|
At each round of conversation, the user will give you the current situation:
|
|
User Query: ...
|
|
Hints: ...
|
|
Your Q&A: ...
|
|
Your work progress: ...
|
|
Evaluation: Evaluation of the latest action and observation
|
|
Suggestion: ...
|
|
|
|
You should consider the following guidelines:
|
|
- Do not create any table in the database
|
|
- Column name can be the same in different tables. Refer to column comments to get more details by using TABLEINFO function
|
|
- A junction table can be used to link tables together. Another use case is for filtering data.
|
|
- If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer.
|
|
- If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query".
|
|
- Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
|
|
|
|
You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action:
|
|
1) Understanding:
|
|
- State your understanding about the current situation.
|
|
2) Reasoning:
|
|
- State your step by step reasoning about the current situation.
|
|
3) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
|
|
4) Action_name (Must be aligned with your plan): Can be one of the following functions:
|
|
- TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. "list_of_table_name" is a list of table name you want to get info. e.g. TABLEINFO["table name 1", "table name 2"]
|
|
- GETDATA[SQL], which you can use to get the data from the database. "SQL" is the single SQL command to be executed against the database.
|
|
For more effective text search, it's advisable to use case-insensitivity and the ILIKE operator.
|
|
Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
|
|
5) Action_input: Input to the action
|
|
6) Observation: Result of the immediately preceding action
|
|
|
|
You should only respond in format as described below:
|
|
Understanding: ...
|
|
Reasoning: ...
|
|
Plan: ...
|
|
Action_name: ...
|
|
Action_input: ...
|
|
Observation: ...
|
|
|
|
Let's begin!
|
|
"""
|
|
|
|
workprogress = ""
|
|
for (k, v) in state[:thoughtHistory]
|
|
if k ∉ [:query]
|
|
workprogress *= "$k: $v\n"
|
|
end
|
|
end
|
|
|
|
usermsg =
|
|
"""
|
|
$(context[:tablelist])
|
|
User query: $(state[:thoughtHistory][:question])
|
|
Hints: $similarSQL
|
|
Your Q&A: $QandA
|
|
Your work progress: $workprogress
|
|
Evaluation: $(state[:evaluation])
|
|
Suggestion: $(state[:suggestion])
|
|
"""
|
|
|
|
_prompt =
|
|
[
|
|
Dict(:name=> "system", :text=> systemmsg),
|
|
Dict(:name=> "user", :text=> usermsg)
|
|
]
|
|
|
|
# put in model format
|
|
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
|
|
prompt *=
|
|
"""
|
|
<|start_header_id|>assistant<|end_header_id|>
|
|
"""
|
|
response = nothing # store for show when error msg show up
|
|
for attempt in 1:10
|
|
try
|
|
response = text2textInstructLLM(prompt)
|
|
|
|
# textToDict() search for action_input
|
|
responsedict = GeneralUtils.textToDict(response,
|
|
["Understanding", "Reasoning", "Plan", "Action_name", "Action_input", "Observation"],
|
|
rightmarker=":", symbolkey=true, lowercasekey=true)
|
|
|
|
delete!(responsedict, :observation)
|
|
|
|
toollist = ["TABLEINFO", "GETDATA"]
|
|
if responsedict[:action_name] ∉ toollist
|
|
error("decisionMaker didn't use the given functions ", @__FILE__, " ", @__LINE__)
|
|
end
|
|
|
|
for i in toollist
|
|
if occursin(i, responsedict[:action_input])
|
|
error("Action_name is in action_input which is not allowed.")
|
|
end
|
|
end
|
|
|
|
for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
|
|
if length(JSON3.write(responsedict[i])) == 0
|
|
error("$i is empty ", @__FILE__, " ", @__LINE__)
|
|
end
|
|
end
|
|
|
|
# check if there are more than 1 key per categories
|
|
for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
|
|
matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
|
|
if length(matchkeys) > 1
|
|
error("DecisionMaker has more than one key per categories")
|
|
end
|
|
end
|
|
|
|
return responsedict
|
|
catch e
|
|
io = IOBuffer()
|
|
showerror(io, e)
|
|
errorMsg = String(take!(io))
|
|
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
|
println("")
|
|
println("Attempt $attempt. Error occurred: $errorMsg\n$st")
|
|
println("")
|
|
end
|
|
end
|
|
error("DecisionMaker failed to generate a thought ", response)
|
|
end
|
|
|
|
# function decisionMaker(state::T2, config::T1
|
|
# )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:AbstractDict}
|
|
|
|
# if isfile("lesson.json")
|
|
# lessonDict = copy(JSON3.read("lesson.json"))
|
|
# else
|
|
# lessonDict = nothing
|
|
# end
|
|
|
|
# lesson =
|
|
# if lessonDict === nothing
|
|
# ""
|
|
# else
|
|
# """
|
|
# You have attempted to help the user before and failed, either because your reasoning for the
|
|
# recommendation was incorrect or your response did not exactly match the user expectation.
|
|
# The following lesson(s) give a plan to avoid failing to help the user in the same way you
|
|
# did previously. Use them to improve your strategy to help the user.
|
|
|
|
# Here are some lessons in JSON format:
|
|
# $(JSON3.write(lessonDict))
|
|
|
|
# When providing the thought and action for the current trial, that into account these failed
|
|
# trajectories and make sure not to repeat the same mistakes and incorrect answers.
|
|
# """
|
|
# end
|
|
|
|
# _prompt =
|
|
# """
|
|
# You are a helpful data engineer.
|
|
# Your goal is to help the user to get what the user wants.
|
|
# You are also keen to improve your helpfulness with lesson(s).
|
|
|
|
# You must follow the following criteria:
|
|
# 1) Get to know what table are available in the database.
|
|
# 2) Get to know what the data in the table looks like.
|
|
# 3) If you can't find a single table that can be used to answer the user's question, try joining multiple tables to see if you can obtain the answer.
|
|
# 4) If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your question".
|
|
|
|
# You should only respond with interleaving Thought, Action, Observation steps.
|
|
# Thought can reason about the current situation, and Action can be one of the following functions:
|
|
# 1) listalltables[NA], which you can use to list all tables in the database and see their descriptions. "NA" word is the function input.
|
|
# 2) tableinfo[table_name], which you can use to see the table and its column description. "table_name" is name of the table you want to get info.
|
|
# 3) getdata[instructions], which you can use to ask other people to get the data from tables for you. "instructions" should clearly describe how you want others to extract the data.
|
|
# For example,
|
|
# a. Query the "Engine" table to identify the engine types that have 3 cylinders. This can be done using a SELECT statement in SQL, filtering the results where the number of cylinders equals 3.
|
|
# b. Once you have identified the engine types with 3 cylinders, use this information to query the "Car" table. You're looking for car models that are associated with these engine types. This can be achieved by performing a JOIN operation between the "Car" and "Engine" tables based on the engine type.
|
|
# 4) finalanswerbox[answer], which returns your answer to the user. "answer" is your answer to the user question.
|
|
# After each observation, provide the next Thought and next Action.
|
|
|
|
# You should only respond in JSON format as describe below:
|
|
# {
|
|
# "thought": "your reasoning",
|
|
# "action": {"name": "action to take", "input": "Action input"},
|
|
# "observation": "result of the action"
|
|
# }
|
|
|
|
# Here are some examples:
|
|
# {
|
|
# "question": "I would like to buy a sedan with 8 seats.",
|
|
# "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.",
|
|
# "action_1": {"name": "inventory", "input": "sedan with 8 seats."},
|
|
# "observation_1": "Several model has 8 seats. Available color are black, red green"
|
|
# }
|
|
# {
|
|
# "thought": "I have a few color for the user to choose from. I will ask him what color he likes.",
|
|
# "action": {"name": "chatbox", "input": "Which color do you like?"}
|
|
# "observation": "I'll take black."
|
|
# }
|
|
|
|
# $lesson
|
|
|
|
# Let's begin!
|
|
|
|
# $(JSON3.write(state[:thoughtHistory]))
|
|
# {"thought"
|
|
# """
|
|
|
|
# # _prompt =
|
|
# # """
|
|
# # You are a helpful data engineer.
|
|
# # Your goal is to help the user to get what the user wants.
|
|
# # You are also keen to improve your helpfulness with lesson(s).
|
|
|
|
# # You must follow the following criteria:
|
|
# # 1) Get to know what table are available in the database.
|
|
# # 2) Get to know what the data in the table looks like.
|
|
# # 3) If you can't find a single table that can be used to answer the user's question, try joining multiple tables to see if you can obtain the answer.
|
|
# # 4) Keep trying even if you get SQL execution error.
|
|
|
|
# # You should only respond with interleaving Thought, Action, Observation steps.
|
|
# # Thought can reason about the current situation, and Action can be one of the following functions:
|
|
# # 1) listalltables[NA], which you can use to list all tables in the database and see their descriptions. "NA" word is the function input.
|
|
# # 2) tableinfo[table_name], which you can use to see the table and its column description. "table_name" is name of the table you want to get info.
|
|
# # 3) getdata[SQL], which you can use to ask other people to get the data from tables for you. "SQL" is the command you will use to extract the data.
|
|
# # 4) finalanswerbox[answer], which returns your answer to the user. "answer" is your answer to the user question.
|
|
# # After each observation, provide the next Thought and next Action.
|
|
|
|
# # You should only respond in JSON format as describe below:
|
|
# # {
|
|
# # "thought": "your reasoning",
|
|
# # "action": {"name": "action to take", "input": "Action input"},
|
|
# # "observation": "result of the action"
|
|
# # }
|
|
|
|
# # Here are some examples:
|
|
# # {
|
|
# # "question": "I would like to buy a sedan with 8 seats.",
|
|
# # "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.",
|
|
# # "action_1": {"name": "inventory", "input": "sedan with 8 seats."},
|
|
# # "observation_1": "Several model has 8 seats. Available color are black, red green"
|
|
# # }
|
|
# # {
|
|
# # "thought": "I have a few color for the user to choose from. I will ask him what color he likes.",
|
|
# # "action": {"name": "chatbox", "input": "Which color do you like?"}
|
|
# # "observation": "I'll take black."
|
|
# # }
|
|
|
|
# # $lesson
|
|
|
|
# # Let's begin!
|
|
|
|
# # $(JSON3.write(state[:thoughtHistory]))
|
|
# # {"thought"
|
|
# # """
|
|
|
|
# # apply LLM specific instruct format
|
|
# externalService = config[:externalservice][:text2textinstruct]
|
|
# llminfo = externalService[:llminfo]
|
|
# prompt =
|
|
# if llminfo[:name] == "llama3instruct"
|
|
# GeneralUtils.formatLLMtext_llama3instruct("system", _prompt)
|
|
# else
|
|
# error("llm model name is not defied yet $(@__LINE__)")
|
|
# end
|
|
|
|
# msgMeta = GeneralUtils.generate_msgMeta(
|
|
# externalService[:mqtttopic],
|
|
# senderName= "decisionMaker",
|
|
# senderId= string(uuid4()),
|
|
# receiverName= "text2textinstruct",
|
|
# mqttBroker= config[:mqttServerInfo][:broker],
|
|
# mqttBrokerPort= config[:mqttServerInfo][:port],
|
|
# )
|
|
|
|
# outgoingMsg = Dict(
|
|
# :msgMeta=> msgMeta,
|
|
# :payload=> Dict(
|
|
# :text=> prompt,
|
|
# :kwargs=> Dict(
|
|
# :max_tokens=> 512,
|
|
# :stop=> ["<|eot_id|>"],
|
|
# )
|
|
# )
|
|
# )
|
|
# @show outgoingMsg
|
|
|
|
# for attempt in 1:5
|
|
# try
|
|
# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
|
# _responseJsonStr = response[:response][:text]
|
|
# expectedJsonExample =
|
|
# """
|
|
# Here is an expected JSON format:
|
|
# {
|
|
# "thought": "...",
|
|
# "action": {"name": "...", "input": "..."},
|
|
# "observation": "..."
|
|
# }
|
|
# """
|
|
# responseJsonStr = FormatCorrector.jsoncorrection(config, _responseJsonStr, expectedJsonExample)
|
|
# thoughtDict = copy(JSON3.read(responseJsonStr))
|
|
|
|
# # check if dict has all required value
|
|
# thought::AbstractString = thoughtDict[:thought]
|
|
# actionname::AbstractString = thoughtDict[:action][:name]
|
|
# actioninput::AbstractString = thoughtDict[:action][:input]
|
|
# if actionname ∈ ["listalltables", "tableinfo", "getdata", "finalanswerbox"]
|
|
# # LLM use available function
|
|
# elseif thought == ""
|
|
# error("DecisionMaker has no thought")
|
|
# elseif length(actioninput) == 0
|
|
# error("DecisionMaker has no actioninput")
|
|
# else
|
|
# error("DecisionMaker use wrong function")
|
|
# end
|
|
|
|
# # check if there are more than 1 key per categories
|
|
# for i ∈ ["thought", "action", "observation"]
|
|
# matchkeys = GeneralUtils.findMatchingDictKey(thoughtDict, i)
|
|
# if length(matchkeys) > 1
|
|
# error("DecisionMaker has more than one key per categories")
|
|
# end
|
|
# end
|
|
|
|
# return thoughtDict
|
|
# catch e
|
|
# io = IOBuffer()
|
|
# showerror(io, e)
|
|
# errorMsg = String(take!(io))
|
|
# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
|
# println("")
|
|
# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
|
|
# println("")
|
|
# end
|
|
# end
|
|
# error("DecisionMaker failed to generate a thought")
|
|
# end
|
|
|
|
|
|
""" Assigns a scalar value to each new child node to be used for selec-
|
|
tion and backpropagation. This value effectively quantifies the agent's progress in task completion,
|
|
serving as a heuristic to steer the search algorithm towards the most promising regions of the tree.
|
|
|
|
# Arguments
|
|
- `state<:AbstractDict`
|
|
one of Yiem's agent
|
|
- `text2textInstructLLM::Function`
|
|
A function that handles communication to LLM service
|
|
|
|
# Return
|
|
- `score::Integer`
|
|
|
|
# Example
|
|
```jldoctest
|
|
julia>
|
|
```
|
|
|
|
# Signature
|
|
"""
|
|
function evaluator(state::T1, text2textInstructLLM::Function;
|
|
addSQLVectorDB::Union{Function, Nothing}=nothing
|
|
) where {T1<:AbstractDict}
|
|
|
|
# systemmsg =
|
|
# """
|
|
# You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions.
|
|
|
|
# Definitions:
|
|
# "question" is the user's question.
|
|
# "thought" is step-by-step reasoning about the current situation.
|
|
# "plan" is what to do to complete the task from the current situation.
|
|
# "action" is the taken action which can be one of the following functions:
|
|
# 1) TABLEINFO[list_of_table_name], which you can use to get the data type of a table column.
|
|
# 2) GETDATA[instruction], which you can use to get the data from the database.
|
|
# 3) ANSWERBOX[answer], which returns your answer to the user. "answer" is your answer to the user question.
|
|
# "observation" is result of the action in JSON format.
|
|
|
|
# At each round of conversation, the user will give you:
|
|
# Context: ...
|
|
# Trajectories: ...
|
|
|
|
# You should then respond to the user with:
|
|
# - Original_question: Repeat the original question.
|
|
# - Evaluation (you must evaluate all of the following points):
|
|
# 1) Analyze the trajectories of a solution to answer the user's original question.
|
|
# Given a question and a trajectory, evaluate its correctness and provide your reasoning and
|
|
# analysis in detail. Focus on the latest thought, action, and observation.
|
|
# Incomplete trajectories can be correct if the thoughts and actions so far are correct,
|
|
# even if the answer is not found yet. Do not generate additional thoughts or actions.
|
|
# 2) How the observation addresses the original question?
|
|
# 3) Provide suggestion (if applicable).
|
|
# - Score: Correctness score s where s is an integer from 0 to 10.
|
|
# - Accepted_as_answer: Decide whether to accept the observation as the answer to the original question.
|
|
# 1) The accepted observation should directly answer the question.
|
|
# 2) The possible responses are either 'Yes' or 'No.'
|
|
|
|
# You should only respond in JSON format as described below:
|
|
# {"original_question": ..., "evaluation": ..., "score": ..., "accepted_as_answer": ...}
|
|
|
|
# Here are correct trajectory examples:
|
|
# user:
|
|
# {
|
|
# "question": "I'm looking for a sedan with an automatic driving feature.",
|
|
# "thought_1": "I have many types of sedans in my inventory, each with diverse features.",
|
|
# "thought_2": "I should check our inventory first to see if we have the one our customer wants.",
|
|
# "action_1": {"name": "inventory", "input": "a sedan with an automatic driving feature"},
|
|
# "observation_1": "Yiem Model A, Conez Model B"
|
|
# }
|
|
# assistant:
|
|
# {
|
|
# "original_question": "the user is looking for a sedan with an automatic driving feature.",
|
|
# "evaluation": "This trajectory is correct because it is logical to use the INVENTORY function to search for inventory based on the details provided in the question, which could lead to a potential answer. The user is asking whether do you have a sedan with an automatic driving feature and the observation provides a list of sedan models that you have. Thus, it is accepted as the answer.",
|
|
# "score": 10,
|
|
# "accepted_as_answer": "Yes"
|
|
# }
|
|
|
|
# user:
|
|
# {
|
|
# "question": "How many cars that fitted with a stereo we have?",
|
|
# "thought_1": "I have many types of car in my inventory, each with diverse features.",
|
|
# "thought_3": "I should check our inventory.",
|
|
# "action_1": {"name": "inventory", "input": "vehicle with a stereo"},
|
|
# "observation_1": "2015 Conez truck."
|
|
# }
|
|
# assistant:
|
|
# {
|
|
# "evaluation": “This approach is correct. It's reasonable to use the INVENTORY function to search for inventory. However, the query asked for a car but the observation was a truck. Thus it is not accepted as the answer. To improve, make sure to input the correct terms and match the requested criteria accurately.”,
|
|
# "score": 5,
|
|
# "accepted_as_answer": "No"
|
|
# }
|
|
|
|
# Here are incorrect trajectory examples:
|
|
# user:
|
|
# {
|
|
# "question": "I'm looking for a sedan with an automatic driving feature. Do you have it in stock?",
|
|
# "thought_1": "I have many types of sedans in my inventory, each with diverse features.",
|
|
# "thought_2": "I will use SEARCHINTERNET function to search for the car.",
|
|
# "action_1": {"name": "SEARCHINTERNET", "input": "a sedan with an automatic driving feature.},
|
|
# "observation_1": "Teza Model A, Teza Model B"
|
|
# }
|
|
# assistant:
|
|
# {
|
|
# "evaluation": "This trajectory is incorrect. Using the SEARCHINTERNET function to search for a sedan in the Internet is illogical because the question asked for the cars available for sale at your dealership. To improve, ensure that you read the question clearly.",
|
|
# "score": 0,
|
|
# "accepted_as_answer": "No"
|
|
# }
|
|
|
|
# Let's begin!
|
|
# """
|
|
|
|
# systemmsg =
|
|
# """
|
|
# You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions.
|
|
|
|
# Definitions:
|
|
# "question" is the user's question.
|
|
# "thought" is step-by-step reasoning about the current situation.
|
|
# "plan" is what to do to complete the task from the current situation.
|
|
# “action_name” is the name of the action taken, which can be one of the following functions:
|
|
# 1) CHATBOX[text], which you can use to talk with the user. "text" is in verbal English.
|
|
# 2) WINESTOCK[query], which you can use to find info about wine in your inventory. "query" is a search term in verbal English. The best query must includes "budget", "type of wine", "characteristics of wine" and "food pairing".
|
|
# "action_input" is the input to the action
|
|
# "observation" is result of the action.
|
|
|
|
# At each round of conversation, the user will give you:
|
|
# Context: ...
|
|
# Trajectories: ...
|
|
|
|
# You should then respond to the user with:
|
|
# - original_question: Repeat the original question.
|
|
# - evaluation (you must evaluate all of the following points in a single paragraph):
|
|
# 1) Analyze the trajectories of a solution to answer the user's original question.
|
|
# Given a question and a trajectory, evaluate its correctness and provide your reasoning and
|
|
# analysis in detail. Focus on the latest thought, action, and observation.
|
|
# Incomplete trajectories can be correct if the thoughts and actions so far are correct,
|
|
# even if the answer is not found yet. Do not generate additional thoughts or actions.
|
|
# 2) How the observation addresses the question exactly?
|
|
# - accepted_as_answer: Decide whether to accept the observation as the answer to the original question.
|
|
# 1) if the observation's content directly answers the question then just accept it as the answer. Oherwise, it is not. The possible responses are either 'Yes' or 'No.'
|
|
# - score: Correctness score s where s is a single integer between 0 to 9.
|
|
# 1) 0 means the trajectories are incorrect.
|
|
# 2) 9 means the trajectories are correct, and the observation's content directly answers the question.
|
|
# - suggestion: if accepted_as_answer is "No", provide suggestion.
|
|
|
|
# You should only respond in format as described below:
|
|
# original_question: ...
|
|
# evaluation: ...
|
|
# accepted_as_answer: ...
|
|
# score: ...
|
|
# suggestion: ...
|
|
|
|
# Let's begin!
|
|
# """
|
|
|
|
systemmsg =
|
|
"""
|
|
You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions.
|
|
|
|
Definitions:
|
|
"question" is the user's question
|
|
"understanding" is agent's understanding about the current situation
|
|
"reasoning" is agent's step-by-step reasoning about the current situation
|
|
"plan" is agent's plan to complete the task from the current situation
|
|
"action_name" is the name of the action taken, which can be one of the following functions:
|
|
- TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. "list_of_table_name" is a list of table name you want to get info. e.g. TABLEINFO["table name 1", "table name 2"]
|
|
- GETDATA[SQL], which you can use to get the data from the database. "SQL" is the single SQL command to be executed against the database.
|
|
"action_input" is the input to the action
|
|
"observation" is result of the preceding immediate action
|
|
|
|
At each round of conversation, the user will give you:
|
|
Context: ...
|
|
Trajectories: ...
|
|
|
|
You should then respond to the user with:
|
|
1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
|
|
- Evaluate the correctness of each section and the overall trajectory based on the given question.
|
|
- Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation.
|
|
- Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached.
|
|
- Do not generate additional thoughts or actions.
|
|
2) Answer_evaluation:
|
|
- Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question
|
|
- State your rationale
|
|
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No"
|
|
Bad example (The observation didn't answers the question):
|
|
question: Find cars with 4 wheels.
|
|
observation: There are 2 cars in the table.
|
|
Good example (The observation answers the question):
|
|
question: Find cars with a stereo.
|
|
observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black.
|
|
4) Score: Correctness score s where s is a single integer between 0 to 9.
|
|
- 0 (the trajectories are incorrect.)
|
|
- 9 (the trajectories are correct, and the observation's content directly answers the question.)
|
|
5) Suggestion: if accepted_as_answer is "No", provide suggestion.
|
|
|
|
You should only respond in format as described below:
|
|
Trajectory_evaluation: ...
|
|
Answer_evaluation: ...
|
|
Accepted_as_answer: ...
|
|
Score: ...
|
|
Suggestion: ...
|
|
|
|
Let's begin!
|
|
"""
|
|
|
|
thoughthistory = ""
|
|
for (k, v) in state[:thoughtHistory]
|
|
thoughthistory *= "$k: $v\n"
|
|
end
|
|
|
|
noise = ""
|
|
for attempt in 1:5
|
|
usermsg =
|
|
"""
|
|
Trajectories: $thoughthistory
|
|
$noise
|
|
"""
|
|
|
|
_prompt =
|
|
[
|
|
Dict(:name=> "system", :text=> systemmsg),
|
|
Dict(:name=> "user", :text=> usermsg)
|
|
]
|
|
|
|
# put in model format
|
|
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
|
|
prompt *=
|
|
"""
|
|
<|start_header_id|>assistant<|end_header_id|>
|
|
"""
|
|
|
|
try
|
|
response = text2textInstructLLM(prompt)
|
|
responsedict = GeneralUtils.textToDict(response,
|
|
["Trajectory_evaluation", "Answer_evaluation", "Accepted_as_answer", "Score", "Suggestion"];
|
|
rightmarker=":", symbolkey=true, lowercasekey=true)
|
|
|
|
# check if dict has all required value
|
|
trajectoryevaluation_text::AbstractString = responsedict[:trajectory_evaluation]
|
|
answerevaluation_text::AbstractString = responsedict[:answer_evaluation]
|
|
# responsedict[:score] = replace(responsedict[:score], r"\(.*?\)" => "") # remove (...) if there is any.
|
|
responsedict[:score] = responsedict[:score][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
|
|
responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5
|
|
score::Integer = responsedict[:score]
|
|
accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
|
|
suggestion::AbstractString = responsedict[:suggestion]
|
|
|
|
if accepted_as_answer ∉ ["Yes", "No"]
|
|
error("generated accepted_as_answer has wrong format")
|
|
end
|
|
|
|
# add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension)
|
|
state[:evaluation] = "$(responsedict[:trajectory_evaluation]) $(responsedict[:answer_evaluation])"
|
|
state[:evaluationscore] = responsedict[:score]
|
|
state[:accepted_as_answer] = responsedict[:accepted_as_answer]
|
|
state[:suggestion] = responsedict[:suggestion]
|
|
|
|
# mark as terminal state when the answer is achieved
|
|
if accepted_as_answer == "Yes"
|
|
|
|
# add to vectorDB only if the answer is achieved and the state is terminal
|
|
# (found some row in the database)
|
|
if addSQLVectorDB !== nothing && state[:isterminal] == true
|
|
addSQLVectorDB(state)
|
|
end
|
|
|
|
# mark the state as terminal state because the evaluation say so.
|
|
state[:isterminal] = true
|
|
|
|
# evaluation score as reward because different answers hold different value for the user.
|
|
state[:reward] = responsedict[:score]
|
|
end
|
|
println("\n~~~ Evaluator() ", @__FILE__, " ", @__LINE__)
|
|
pprintln(Dict(responsedict))
|
|
|
|
return responsedict[:score]
|
|
catch e
|
|
io = IOBuffer()
|
|
showerror(io, e)
|
|
errorMsg = String(take!(io))
|
|
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
|
println("")
|
|
println("Attempt $attempt. Error occurred: $errorMsg\n$st")
|
|
println("")
|
|
noise = GeneralUtils.randstrings(3, 5)
|
|
end
|
|
end
|
|
error("evaluator failed to generate an evaluation")
|
|
end
|
|
|
|
|
|
"""
|
|
|
|
# Arguments
|
|
|
|
# Return
|
|
|
|
# Example
|
|
```jldoctest
|
|
julia>
|
|
```
|
|
|
|
# TODO
|
|
- [] update docstring
|
|
- [] implement the function
|
|
- [x] add try block. check result that it is expected before returning
|
|
|
|
# Signature
|
|
"""
|
|
function reflector(config::T1, state::T2)::String where {T1<:AbstractDict, T2<:AbstractDict}
|
|
# https://github.com/andyz245/LanguageAgentTreeSearch/blob/main/hotpot/hotpot.py
|
|
|
|
systemmsg =
|
|
"""
|
|
You will be given a question and a trajectory of the previous help you've done for a user.
|
|
You were unsuccessful in helping the user either because you use the wrong syntax, or use the wrong function, or refer to item that don't exist in the database.
|
|
In a few sentences, Diagnose a possible reason for failure and devise a new, specific and concise lesson that aims to mitigate the same failure.
|
|
Use complete sentences.
|
|
|
|
You should only respond in JSON format as describe below:
|
|
{"reflection": "your relection"}
|
|
|
|
Here are some examples:
|
|
user:
|
|
{
|
|
"question": "Hello, I would like a get a bottle of wine",
|
|
"thought_1": "A customer wants to buy a bottle of wine. Before making a recommendation, I need to know more about their preferences.",
|
|
"action_1": {"name": "chatbox", "input": "What is the occasion for which you're buying this wine?"},
|
|
"observation_1": "We are holding a wedding party",
|
|
|
|
"thought_2": "A wedding party, that's a great occasion! The customer might be looking for a celebratory drink. Let me ask some more questions to narrow down the options.",
|
|
"action_2": {"name": "chatbox", "input": "What type of food will you be serving at the wedding?"},
|
|
"observation_2": "It will be Thai dishes.",
|
|
|
|
"thought_3": "With Thai food, I should recommend a wine that complements its spicy and savory flavors. And since it's a celebratory occasion, the customer might prefer a full-bodied wine.",
|
|
"action_3": {"name": "chatbox", "input": "What is your budget for this bottle of wine?"},
|
|
"observation_3": "I would spend up to 50 bucks.",
|
|
|
|
"thought_4": "Now that I have some more information, it's time to narrow down the options.",
|
|
"action_4": {"name": "winestock", "input": "red wine with full body, pairs well with spicy food, budget \$50"},
|
|
"observation_4": "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n",
|
|
|
|
"thought_5": "Now that I have a list of potential wines, I need to know more about the customer's taste preferences.",
|
|
"action_5": {"name": "chatbox", "input": "What type of wine characteristics are you looking for? (e.g. t.e.g. tannin level, sweetness, intensity, acidity)"},
|
|
"observation_5": "I like full-bodied red wine with low tannin.",
|
|
|
|
"thought_6": "Now that I have more information about the customer's preferences, it's time to make a recommendation.",
|
|
"action_6": {"name": "recommendbox", "input": "El Enemigo Cabernet Franc 2019"},
|
|
"observation_6": "I don't like the one you recommend. I want dry wine."
|
|
}
|
|
assistant:
|
|
{
|
|
"reflection": "I asked the user about the occasion, food type, and budget, and then searched for wine in the inventory right away. However, I should have asked the user for the specific wine type and their preferences in order to gather more information before making a recommendation."
|
|
}
|
|
|
|
user:
|
|
{
|
|
"question": "How many wines suitable to be paired with lamb?",
|
|
"thought_1": "The user wants to know how many wines that can be paired with lamb, I will try to find the table that has information about pairing between wines and food items.",
|
|
"action_1": {"name": "getdata", "input": "What is the occasion for which you're buying this wine?"},
|
|
"observation_1": "We are holding a wedding party",
|
|
|
|
"thought_2": "A wedding party, that's a great occasion! The customer might be looking for a celebratory drink. Let me ask some more questions to narrow down the options.",
|
|
"action_2": {"name": "chatbox", "input": "SELECT * FROM wine_food WHERE obj_description LIKE '%lamb%'"},
|
|
"observation_2": "SQL execution error: SQL syntax error. It must end with character ';'",
|
|
}
|
|
assistant:
|
|
{
|
|
"reflection": "I need to have ';' at the end of the SQL query."
|
|
}
|
|
|
|
Let's begin!
|
|
"""
|
|
|
|
usermsg =
|
|
"""
|
|
$(JSON3.write(state[:thoughtHistory]))
|
|
"""
|
|
|
|
_prompt =
|
|
[
|
|
Dict(:name=> "system", :text=> systemmsg),
|
|
Dict(:name=> "user", :text=> usermsg)
|
|
]
|
|
|
|
# put in model format
|
|
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
|
|
prompt *=
|
|
"""
|
|
<|start_header_id|>assistant<|end_header_id|>
|
|
"""
|
|
|
|
externalService = config[:externalservice][:text2textinstruct]
|
|
|
|
|
|
# apply LLM specific instruct format
|
|
externalService = config[:externalservice][:text2textinstruct]
|
|
|
|
msgMeta = GeneralUtils.generate_msgMeta(
|
|
externalService[:mqtttopic],
|
|
senderName= "reflector",
|
|
senderId= string(uuid4()),
|
|
receiverName= "text2textinstruct",
|
|
mqttBroker= config[:mqttServerInfo][:broker],
|
|
mqttBrokerPort= config[:mqttServerInfo][:port],
|
|
)
|
|
|
|
outgoingMsg = Dict(
|
|
:msgMeta=> msgMeta,
|
|
:payload=> Dict(
|
|
:text=> prompt,
|
|
:kwargs=> Dict(
|
|
:max_tokens=> 512,
|
|
:stop=> ["<|eot_id|>"],
|
|
)
|
|
)
|
|
)
|
|
|
|
for attempt in 1:5
|
|
try
|
|
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
|
_responseJsonStr = response[:response][:text]
|
|
expectedJsonExample =
|
|
"""
|
|
Here is an expected JSON format:
|
|
{"reflection": "..."}
|
|
"""
|
|
# responseJsonStr, errormsg, success =
|
|
# FormatCorrector.jsoncorrection(config, _responseJsonStr, expectedJsonExample)
|
|
if !success
|
|
error("Not valid JSON")
|
|
end
|
|
|
|
reflectionDict = copy(JSON3.read(responseJsonStr))
|
|
|
|
# check if dict has all required value
|
|
dummya::AbstractString = reflectionDict[:reflection]
|
|
|
|
return reflectionDict[:reflection]
|
|
catch e
|
|
io = IOBuffer()
|
|
showerror(io, e)
|
|
errorMsg = String(take!(io))
|
|
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
|
println("")
|
|
@warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
|
|
println("")
|
|
end
|
|
end
|
|
error("reflector failed to generate a thought")
|
|
end
|
|
|
|
|
|
""" Get a new state
|
|
|
|
# Arguments
|
|
- `state<:AbstractDict`
|
|
state's dictionary
|
|
- `args::NamedTuple`
|
|
Arguments for decisionMaker() and others
|
|
|
|
# Return
|
|
- `NamedTuple{(:newNodeKey, :newstate, :progressvalue), Tuple{String, T, Integer}}`
|
|
|
|
# Example
|
|
```jldoctest
|
|
julia> using SQLLLM, DataStructures
|
|
julia> state = Dict(
|
|
:isterminal => false,
|
|
:lesson => nothing,
|
|
:reward => 0,
|
|
:evaluation => "None",
|
|
:accepted_as_answer => "No",
|
|
:thoughtHistory => OrderedDict{Symbol, Any}(:question => "How many wines do you have that can be paired with lamb?"),
|
|
:evaluationscore => 0,
|
|
:suggestion => "None"
|
|
)
|
|
```
|
|
|
|
# TODO
|
|
- [] add embedding of newstate and store in newstate[:embedding]
|
|
- [WORKING] should getdata() return isterminal?
|
|
# Signature
|
|
"""
|
|
function transition(state::T, args::NamedTuple
|
|
)::NamedTuple{(:newNodeKey, :newstate, :progressvalue), Tuple{String, T, Integer}} where {T<:AbstractDict}
|
|
|
|
decisionMakerF::Function = args[:decisionMaker]
|
|
evaluatorF::Function = args[:evaluator]
|
|
reflector::Function = args[:reflector]
|
|
context = args[:context]
|
|
executeSQLF::Function = args[:executeSQL]
|
|
text2textInstructLLM::Function = args[:text2textInstructLLM]
|
|
addSQLVectorDBF::Function = args[:addSQLVectorDB]
|
|
querySQLVectorDBF::Function = args[:querySQLVectorDB]
|
|
|
|
# find similar SQL statement
|
|
similarSQL = querySQLVectorDBF(state)
|
|
|
|
QandA = generatequestion(state, context, text2textInstructLLM; similarSQL=similarSQL)
|
|
# getting SQL from vectorDB
|
|
thoughtDict = decisionMakerF(state, context, text2textInstructLLM, QandA; similarSQL=similarSQL)
|
|
actionname = thoughtDict[:action_name]
|
|
actioninput = thoughtDict[:action_input]
|
|
|
|
# map action and input() to llm function
|
|
response =
|
|
if actionname == "listalltables"
|
|
# deepcopy(state[:virtualCustomerChatHistory]) because I want to keep it clean
|
|
# so that other simulation start from this same node is not contaminated with actioninput
|
|
listAllTable_json(executeSQLF)
|
|
elseif actionname == "TABLEINFO"
|
|
input = copy(JSON3.read(actioninput))
|
|
tableinfo(executeSQLF, input)
|
|
elseif actionname == "GETDATA"
|
|
userintention = Dict(:userintention=> "$(thoughtDict[:plan]) $(thoughtDict[:plan])")
|
|
getdata(actioninput, userintention, executeSQLF, text2textInstructLLM)
|
|
else
|
|
error("undefined LLM function. Requesting $actionname")
|
|
end
|
|
# this section allow LLM functions above to have different return values.
|
|
success::Bool = haskey(response, :success) ? response[:success] : false
|
|
result = success ? response[:result] : response[:errormsg]
|
|
select = haskey(response, :select) ? response[:select] : nothing
|
|
reward::Integer = haskey(response, :reward) ? response[:reward] : 0
|
|
isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false
|
|
newNodeKey, newstate = makeNewState(state, thoughtDict, JSON3.write(result), select, reward, isterminal)
|
|
progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM;
|
|
addSQLVectorDB=addSQLVectorDBF)
|
|
|
|
return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue)
|
|
end
|
|
|
|
|
|
""" Ask the database using English language.
|
|
|
|
# Arguments
|
|
- `query<:AbstractString`
|
|
a query
|
|
- `executeSQL::Function`
|
|
a connection object to a database
|
|
- `text2textInstructLLM::Function`
|
|
A function that handles communication to text2text instruct LLM service.
|
|
|
|
# Return
|
|
- `resulttext::String`
|
|
The result of the query in English.
|
|
|
|
# Example
|
|
```jldoctest
|
|
julia> using LibPQ, JSON3, UUIDs
|
|
julia> using SQLLLM, GeneralUtils
|
|
julia> function executeSQL(sql)
|
|
DBconnection = LibPQ.Connection("host=192.168.88.122 port=5432 dbname=xyz user=zyx password=1234")
|
|
result = LibPQ.execute(DBconnection, sql)
|
|
close(DBconnection)
|
|
return result
|
|
end
|
|
julia> function text2textInstructLLM(prompt::String)
|
|
config = Dict(
|
|
:mqttServerInfo => Dict(
|
|
:description => "mqtt server info",
|
|
:port => 1883,
|
|
:broker => "mqtt.yiem.cc"
|
|
),
|
|
:externalservice => Dict(
|
|
:text2textinstruct => Dict(
|
|
:mqtttopic => "/loadbalancer/requestingservice",
|
|
:description => "text to text service with instruct LLM",
|
|
:llminfo => Dict(:name => "llama3instruct")
|
|
),
|
|
)
|
|
)
|
|
|
|
# apply LLM specific instruct format
|
|
externalService = config[:externalservice][:text2textinstruct]
|
|
|
|
msgMeta = GeneralUtils.generate_msgMeta(
|
|
externalService[:mqtttopic],
|
|
senderName= "SQLLLM",
|
|
senderId= string(uuid4()),
|
|
receiverName= "text2textinstruct",
|
|
mqttBroker= config[:mqttServerInfo][:broker],
|
|
mqttBrokerPort= config[:mqttServerInfo][:port],
|
|
)
|
|
|
|
outgoingMsg = Dict(
|
|
:msgMeta=> msgMeta,
|
|
:payload=> Dict(
|
|
:text=> prompt,
|
|
:kwargs=> Dict(
|
|
:max_tokens=> 512,
|
|
:stop=> ["<|eot_id|>"],
|
|
:temperature=> 0.2,
|
|
)
|
|
)
|
|
)
|
|
|
|
_response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
|
response = _response[:response][:text]
|
|
|
|
return response
|
|
end
|
|
julia> query = Dict(:text=> "How many wines do you have that can be paired with lamb?")
|
|
julia> result = SQLLLM.query(query, executeSQL, text2textInstructLLM)
|
|
julia> println(result)
|
|
```
|
|
|
|
# Signature
|
|
"""
|
|
function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
|
addSQLVectorDB::Union{Function, Nothing}=nothing,
|
|
querySQLVectorDB::Union{Function, Nothing}=nothing
|
|
)::String where {T<:AbstractString}
|
|
# add extra context for Evaluator so that it knows the observation is from seaching a database
|
|
query = "Search the database for {$query}"
|
|
initialstate = Dict{Symbol, Any}(
|
|
:reward=> 0,
|
|
:isterminal=> false,
|
|
:evaluation=> "None",
|
|
:evaluationscore=> 0,
|
|
:suggestion=> "None",
|
|
:accepted_as_answer=> "No",
|
|
:lesson=> nothing,
|
|
|
|
# contain question, thought_1, action_1, observation_1, thought_2, ...
|
|
:thoughtHistory=> OrderedDict{Symbol, Any}(
|
|
#[] :recap=>,
|
|
:question=> query,
|
|
),
|
|
)
|
|
context = Dict(
|
|
:tablelist => listAllTable_str(executeSQL)[:result]
|
|
)
|
|
transitionargs = (
|
|
decisionMaker=decisionMaker,
|
|
evaluator=evaluator,
|
|
reflector=reflector,
|
|
context=context,
|
|
executeSQL=executeSQL,
|
|
text2textInstructLLM=text2textInstructLLM,
|
|
querySQLVectorDB=querySQLVectorDB,
|
|
addSQLVectorDB=addSQLVectorDB,
|
|
)
|
|
|
|
earlystop(state) = state[:reward] >= 8 ? true : false
|
|
|
|
_, resultState = LLMMCTS.runMCTS(initialstate, transition, transitionargs;
|
|
totalsample=1, maxdepth=3, maxiterations=3, explorationweight=1.0,
|
|
earlystop=earlystop)
|
|
latestKey, _ = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation")
|
|
resulttext = resultState[:thoughtHistory][latestKey]
|
|
|
|
return resulttext
|
|
end
|
|
|
|
|
|
""" Make a new state.
|
|
|
|
# Arguments
|
|
|
|
# Return
|
|
|
|
# Example
|
|
```jldoctest
|
|
julia>
|
|
```
|
|
|
|
# Signature
|
|
"""
|
|
function makeNewState(currentstate::T1, thoughtDict::T4, response::T2, select::Union{T3, Nothing},
|
|
reward::T3, isterminal::Bool
|
|
)::NamedTuple{(:newNodeKey, :newstate), Tuple{String, Dict{Symbol, <:Any}}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
|
|
|
|
keys = [:understanding, :reasoning, :action_name, :action_input, :observation]
|
|
# latestKeys = []
|
|
|
|
currentstate_latestKey, currentstate_latestIndice =
|
|
GeneralUtils.findHighestIndexKey(currentstate[:thoughtHistory], keys[1])
|
|
nextindice = currentstate_latestKey !== nothing ? currentstate_latestIndice + 1 : 1
|
|
# currentstate_latestKey == :NA ? 1 : currentstate_latestIndice + 1
|
|
|
|
currentstate_latestKey = makeNextKey.(keys, nextindice)
|
|
|
|
# add Thought, action, observation to thoughtHistory
|
|
newstate = deepcopy(currentstate)
|
|
for (x, y) in zip(keys, currentstate_latestKey)
|
|
if x != :observation
|
|
newstate[:thoughtHistory][y] = thoughtDict[Symbol(x)]
|
|
else
|
|
newstate[:thoughtHistory][y] = response
|
|
end
|
|
end
|
|
|
|
newstate[:reward] = reward
|
|
newstate[:select] = select
|
|
newstate[:isterminal] = isterminal
|
|
|
|
newNodeKey = GeneralUtils.uuid4snakecase()
|
|
|
|
return (newNodeKey=newNodeKey, newstate=newstate)
|
|
end
|
|
|
|
|
|
makeNextKey(key, indice) = Symbol("$(key)_$indice")
|
|
|
|
|
|
function generatequestion(state::T1, context, text2textInstructLLM::Function;
|
|
similarSQL::Union{T2, Nothing}=nothing
|
|
)::String where {T1<:AbstractDict, T2<:AbstractString}
|
|
|
|
similarSQL =
|
|
if similarSQL === nothing
|
|
"None"
|
|
else
|
|
"This is the closest matching SQL statement for a similar query: $similarSQL"
|
|
end
|
|
|
|
systemmsg =
|
|
"""
|
|
You are a helpful assistant that generate multiple questions about the current situation.
|
|
|
|
At each round of conversation, the user will give you the current situation:
|
|
User query: ...
|
|
Hints: ...
|
|
Your work progress: ...
|
|
|
|
About the tables in the database:
|
|
- Column name can be the same in different tables. Refer to column comments to get more details.
|
|
- Columns represent properties of the items the table represents. For example, the 'color' column in a "dealer_car" table corresponds to the color of the dealer's car.
|
|
- A junction table can be used to link tables together.
|
|
|
|
You must follow the following guidelines:
|
|
1) Your question must be specific to locating each piece of information mentioned in the query and how to retrieve it.
|
|
2) Your question should be specific, self-contained and not require any additional context.
|
|
3) Some information can be accessed by joining multiple tables.
|
|
4) Do not generate any question or comments at the end.
|
|
|
|
You should then respond to the user with:
|
|
1) Understanding:
|
|
- State your understanding about the current situation.
|
|
2) Q: Given the situation, "ask yourself" about the situation at least five, but no more than ten, questions.
|
|
3) A: Given the situation, "answer to yourself" the best you can.
|
|
|
|
You must only respond in format as described below:
|
|
Understanding: ...
|
|
Q1: ...
|
|
A1: ...
|
|
Q2: ...
|
|
A2: ...
|
|
Q3: ...
|
|
A3: ...
|
|
...
|
|
|
|
Let's begin!
|
|
"""
|
|
|
|
workprogress = ""
|
|
for (k, v) in state[:thoughtHistory]
|
|
if k ∉ [:query]
|
|
workprogress *= "$k: $v\n"
|
|
end
|
|
end
|
|
|
|
response = nothing # store for show when error msg show up
|
|
errornote = ""
|
|
|
|
for attempt in 1:10
|
|
|
|
usermsg =
|
|
"""
|
|
$(context[:tablelist])
|
|
User query: $(state[:thoughtHistory][:question])
|
|
Hints: $similarSQL
|
|
Your work progress: $workprogress
|
|
$errornote
|
|
"""
|
|
|
|
_prompt =
|
|
[
|
|
Dict(:name=> "system", :text=> systemmsg),
|
|
Dict(:name=> "user", :text=> usermsg)
|
|
]
|
|
|
|
# put in model format
|
|
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct")
|
|
prompt *=
|
|
"""
|
|
<|start_header_id|>assistant<|end_header_id|>
|
|
"""
|
|
|
|
try
|
|
response = text2textInstructLLM(prompt)
|
|
q_number = count("Q", response)
|
|
if q_number < 1
|
|
error("too few questions only $q_number questions are generated ", @__FILE__, " ", @__LINE__)
|
|
end
|
|
# response = string(split(response, "Please")[1]) # LLM usually add comments which is no need.
|
|
responsedict = GeneralUtils.textToDict(response,
|
|
["Understanding", "Q1"],
|
|
rightmarker=":", symbolkey=true; lowercasekey=true)
|
|
response = "Q1: " * responsedict[:q1]
|
|
println("--> SQLLLM generatequestion ", @__FILE__, " ", @__LINE__)
|
|
pprintln(Dict(responsedict))
|
|
return response
|
|
catch e
|
|
io = IOBuffer()
|
|
showerror(io, e)
|
|
errorMsg = String(take!(io))
|
|
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
|
println("")
|
|
println("Attempt $attempt. Error occurred: $errorMsg\n$st")
|
|
println("")
|
|
end
|
|
end
|
|
error("generatequestion failed to generate a thought ", response)
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
end # module interface |