ton/SQLLLM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat lamaiin
2025-05-18 17:21:55 +07:00
parent 0a0e36d86a
commit b3537a83e0
1 changed files with 67 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
src/interface.jl
View File

@@ -136,18 +136,13 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
systemmsg =
"""
You are a helpful assistant that find the data from a database to satisfy the user's question.
You are also eager to improve your helpfulness.
You are working under your mentor supervision and you are also eager to improve your helpfulness.
For your information:
- Observation: Result of the immediately preceding action
At each round of conversation, you will be given the following information:
table_schema: schema of tables in the database
most_relevant_SQL: The closest known SQL for this question
query_result_of_most_relevant_SQL: the query result when executing the most_relevant_SQL against a database.
progress: your progress so far
evaluation: Evaluation of the immediately preceding action and observation
suggestion: Suggestion for the immediately preceding action and observation
context: additional information about the current situation
You must follow the following guidelines:
- Keep SQL queries focused only on the provided information.
@@ -160,22 +155,24 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
- If there is no search result from the database, remove the restrictive criteria until a search result is available, and proceed from there.
You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input:
Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
Action_name: (Typically corresponds to the execution of the first step in your plan)
1) plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific.
2) action_name: (Typically corresponds to the execution of the first step in your plan)
Can be one of the following function names:
- RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database.
For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
4) Action_input: Input to the action
3) action_input: Input to the action
You should only respond in format as described below:
Plan: ...
Action_name: ...
Action_input: ...
You should only respond in JSON format as described below:
{
"plan": "...",
"action_name": "...",
"action_input": "..."
}
Let's begin!
"""
requiredKeys = [:plan, :action_name, :action_input]
workprogress = ""
for (k, v) in state[:thoughtHistory]
if k ∉ [:question]
@@ -193,13 +190,6 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
similarSQL_ = sql !== nothing ? sql : "None"
end
header = ["Plan:", "Action_name:", "Action_input:"]
dictkey = ["plan", "action_name", "action_input"]
llmkwargs=Dict(
:num_ctx => 32768,
:temperature => 0.5,
)
for attempt in 1:maxattempt
@@ -208,22 +198,19 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
context =
"""
<context>
<table_schema>
<table_schema> This is schema of tables in the database:
$(additionalinfo[:tablelist])
</table_schema>
<most_relevant_SQL>
<most_relevant_SQL> The closest known SQL for this question is:
$similarSQL_
</most_relevant_SQL>
<query_result_of_most_relevant_SQL>
<query_result_of_most_relevant_SQL> This is the query result when executing the most_relevant_SQL against a database. You can use this to see how the data are stored.
winery: Chateau Montelena, wine_name: The Montelena Estate Cabernet Sauvignon, wine_id: 97264f71-007c-4cce-a3fe-2cc88fba4d05, vintage: 2017, region: Napa Valley, country: United States, wine_type: red, grape: Cabernet Sauvignon, serving_temperature: 15 to 18 Celsius, sweetness: 1, intensity: 5, tannin: 4, acidity: 4, tasting_notes: oak, vanilla, tobacco, blackberry, plum, black cherry, leather, earthy, smoke, price: 19.95, currency: USD
</query_result_of_most_relevant_SQL>
<progress>
<progress> your work progress so far:
$workprogress
</progress>
<evaluation>
$(state[:evaluation])
</evaluation>
<suggestion>
<suggestion> This is your mentor's suggestion for the immediately preceding action and observation
$(state[:suggestion])
</suggestion>
P.S. $errornote
@@ -240,72 +227,40 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName)
# add info
prompt = prompt * context
response = text2textInstructLLM(prompt; llmkwargs=llmkwargs)
response = text2textInstructLLM(prompt)
response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
think, response = GeneralUtils.extractthink(response)
# LLM tends to generate observation given that it is in the input
response =
if occursin("observation:", response)
string(split(response, "observation:")[1])
elseif occursin("Observation:", response)
string(split(response, "Observation:")[1])
elseif occursin("observation_", response)
string(split(response, "observation_")[1])
elseif occursin("Observation_", response)
string(split(response, "Observation_")[1])
else
response
end
# sometime LLM output something like **Comprehension**: which is not expected
response = replace(response, "**"=>"")
response = replace(response, "***"=>"")
# # some time LLM output Plan_1: so we need to detect and replace topic numbering
# regex = r"_[0-1000]+:"
# matches = collect(eachmatch(regex, response))
# for m in matches
# response = replace(response, string(m.match)=>":")
# end
if occursin("NULL", response)
errornote = "\nYour previous attempt was NULL. This is not allowed"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
# # detect if there are more than 1 key per categories
# wordcount = GeneralUtils.countGivenWords(response, header)
# duplicateKeywordFlag = false
# for (i, v) in enumerate(wordcount)
# keyword = header[i]
# keywordNumber = v
# if keywordNumber > 1
# errornote = "\nSQL query has duplicated keyword, $keyword"
# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# duplicateKeywordFlag = true
# break
# end
# end
# duplicateKeywordFlag == true ? continue : nothing
# check whether response has all header
detected_kw = GeneralUtils.detect_keyword(header, response)
if 0 ∈ values(detected_kw)
errornote = "\nYour previous attempt did not have all points according to the required response format"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
elseif sum(values(detected_kw)) > length(header)
errornote = "\nYour previous attempt has duplicated points according to the required response format"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
responsedict = nothing
try
responsedict = copy(JSON3.read(response))
catch
println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
responsedict = GeneralUtils.textToDict(response, header;
dictKey=dictkey, symbolkey=true)
# check whether all answer's key points are in responsedict
_responsedictKey = keys(responsedict)
responsedictKey = [i for i in _responsedictKey] # convert into a list
is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys]
if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys)
errornote = "Your previous attempt has more key points than answer's required key points."
println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
elseif !all(is_requiredKeys_in_responsedictKey)
zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey)
missingkeys = [requiredKeys[i] for i in zeroind]
errornote = "$missingkeys are missing from your previous response"
println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
delete!(responsedict, :observation)
@@ -336,13 +291,13 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
end
end
for i ∈ Symbol.(dictkey)
if length(JSON3.write(responsedict[i])) == 0
errornote = "Your previous attempt has empty value for $i"
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
end
# for i ∈ Symbol.(dictkey)
# if length(JSON3.write(responsedict[i])) == 0
# errornote = "Your previous attempt has empty value for $i"
# println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# continue
# end
# end
println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict))
@@ -634,7 +589,7 @@ julia>
# Signature
"""
function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, llmFormatName::String;
function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String;
maxattempt=10
) where {T1<:AbstractDict}
@@ -653,8 +608,8 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
"observation" is result of the preceding immediate action
At each round of conversation, you will be given the following information:
Trajectory: A history of how you worked on the question chronologically.
table_schema: schema of tables in the database
trajectory: A history of how you worked on the question chronologically
evaluatee_context: The context that evaluatee use to make a decision
You must follow the following guidelines:
- When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
@@ -704,18 +659,20 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll
errornote = "N/A"
for attempt in 1:maxattempt
usermsg =
"""
Trajectory: $thoughthistory
"""
usermsg =
"""
<trajectory>
$thoughthistory
</trajectory>
"""
context =
"""
<context>
<table_schema>
$(additionalinfo[:tablelist])
</table_schema>
P.S. $errornote
</context>
"""
<context>
<evaluatee_context>
thoughtDict[:context]
</evaluatee_context>
P.S. $errornote
</context>
"""
unformatPrompt =
@@ -1056,7 +1013,7 @@ function transition(state::T, args::NamedTuple
reward::Integer = haskey(response, :reward) ? response[:reward] : 0
isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false
newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON3.write(result), select, reward, isterminal)
progressvalue::Integer = evaluatorF(newstate, context, text2textInstructLLM, llmFormatName)
progressvalue::Integer = evaluatorF(newstate, thoughtDict, text2textInstructLLM, llmFormatName)
return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue)
end
@@ -1333,12 +1290,12 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
# JSON3.pretty(io, highValueState)
# end
selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName)
resultState = highValueState[selected] #BUG compareState() select 0
resultState = highValueState[selected]
end
latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation")
action_input = Symbol("action_input_$latestInd") # latest sql
sql = resultState[:thoughtHistory][action_input]
extracted = resultState[:thoughtHistory][latestKey]
extractedTableContent = resultState[:thoughtHistory][latestKey]
# add to vectorDB only if the answer is achieved and the state is terminal
if insertSQLVectorDB !== nothing && resultState[:isterminal] == true &&
@@ -1347,11 +1304,11 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
insertSQLVectorDB(resultState[:thoughtHistory][:question], sql)
end
if extracted === nothing
if extractedTableContent === nothing
println("\nSQLLLM query() return nothing ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
result = (text=extracted, rawresponse=resultState[:rawresponse])
result = (text=extractedTableContent, rawresponse=resultState[:rawresponse])
return result
end