update
This commit is contained in:
narawat lamaiin
@@ -227,11 +227,11 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
|
||||
|
||||
if occursin("NULL", response)
|
||||
errornote = "\nSQL decisionMaker() NULL response is not allowed"
|
||||
println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
|
||||
println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__)
|
||||
continue
|
||||
end
|
||||
|
||||
header = ["Understanding", "Reasoning", "Plan", "Action_name", "Action_input", "Observation"]
|
||||
header = ["Understanding", "Reasoning", "Plan", "Action_name", "Action_input"]
|
||||
|
||||
# detect if there are more than 1 key per categories
|
||||
count = GeneralUtils.countGivenWords(response, header)
|
||||
@@ -241,13 +241,25 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
|
||||
keywordNumber = v
|
||||
if keywordNumber > 1
|
||||
errornote = "\nSQL query has duplicated keyword, $keyword"
|
||||
println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
|
||||
println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__)
|
||||
duplicateKeywordFlag = true
|
||||
break
|
||||
end
|
||||
end
|
||||
duplicateKeywordFlag == true ? continue : nothing
|
||||
|
||||
# check whether response has all header
|
||||
kw = []
|
||||
# use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
|
||||
for keyword in header
|
||||
detected = GeneralUtils.detect_keyword(keyword, response)
|
||||
push!(kw, detected)
|
||||
end
|
||||
if nothing ∈ kw
|
||||
println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__)
|
||||
continue # try again next loop
|
||||
end
|
||||
|
||||
# textToDict() search for action_input
|
||||
responsedict = GeneralUtils.textToDict(response, header,
|
||||
rightmarker=":", symbolkey=true, lowercasekey=true)
|
||||
@@ -269,14 +281,14 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
|
||||
toollist = ["TABLEINFO", "GETDATA"]
|
||||
if responsedict[:action_name] ∉ toollist
|
||||
errornote = "\nYou must only use the given functions"
|
||||
println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
|
||||
println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__)
|
||||
continue
|
||||
end
|
||||
|
||||
for i in toollist
|
||||
if occursin(i, responsedict[:action_input])
|
||||
errornote = "\n action_name is in action_input which is not allowed."
|
||||
println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
|
||||
println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__)
|
||||
continue
|
||||
end
|
||||
end
|
||||
@@ -284,7 +296,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
|
||||
for i ∈ [:understanding, :reasoning, :plan, :action_name, :action_input]
|
||||
if length(JSON3.write(responsedict[i])) == 0
|
||||
errornote = "\n $i is empty"
|
||||
println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
|
||||
println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__)
|
||||
continue
|
||||
end
|
||||
end
|
||||
@@ -294,7 +306,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function,
|
||||
matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
|
||||
if length(matchkeys) > 1
|
||||
errornote = "\n $i has more than one key"
|
||||
println("Attempt $attempt $errornote ", @__FILE__, " ", @__LINE__)
|
||||
println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__)
|
||||
continue
|
||||
end
|
||||
end
|
||||
@@ -481,6 +493,9 @@ function evaluator(state::T1, text2textInstructLLM::Function;
|
||||
At each round of conversation, the user will give you:
|
||||
Context: ...
|
||||
Trajectory: ...
|
||||
|
||||
You must follow the following guidelines:
|
||||
- When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer.
|
||||
|
||||
You should then respond to the user with:
|
||||
1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question.
|
||||
@@ -494,7 +509,7 @@ function evaluator(state::T1, text2textInstructLLM::Function;
|
||||
3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No"
|
||||
Bad example (The observation didn't answers the question):
|
||||
question: Find cars with 4 wheels.
|
||||
observation: There are 2 cars in the table.
|
||||
observation: There are an apple in the table.
|
||||
Good example (The observation answers the question):
|
||||
question: Find cars with a stereo.
|
||||
observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black.
|
||||
@@ -575,7 +590,7 @@ function evaluator(state::T1, text2textInstructLLM::Function;
|
||||
# evaluation score as reward because different answers hold different value for the user.
|
||||
state[:reward] = responsedict[:score]
|
||||
end
|
||||
println("\n~~~ Evaluator() ", @__FILE__, " ", @__LINE__)
|
||||
println("\n~~~ Evaluator() ", @__FILE__, ":", @__LINE__)
|
||||
pprintln(Dict(responsedict))
|
||||
|
||||
return responsedict[:score]
|
||||
@@ -964,7 +979,9 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function;
|
||||
earlystop(state) = state[:reward] >= 8 ? true : false
|
||||
|
||||
_, resultState = LLMMCTS.runMCTS(initialstate, transition, transitionargs;
|
||||
totalsample=1, maxdepth=3, maxiterations=3, explorationweight=1.0,
|
||||
horizontalSampleExpansionPhase=2,
|
||||
horizontalSampleSimulationPhase=1,
|
||||
maxdepth=3, maxiterations=2, explorationweight=1.0,
|
||||
earlystop=earlystop)
|
||||
latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation")
|
||||
action_input = Symbol("action_input_$latestInd") # latest sql
|
||||
@@ -1152,7 +1169,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
|
||||
["Understanding", "Q1"],
|
||||
rightmarker=":", symbolkey=true; lowercasekey=true)
|
||||
response = "Q1: " * responsedict[:q1]
|
||||
println("\n~~~ SQLLLM generatequestion() ", @__FILE__, " ", @__LINE__)
|
||||
println("\n~~~ SQLLLM generatequestion() ", @__FILE__, ":", @__LINE__)
|
||||
pprintln(Dict(responsedict))
|
||||
return response
|
||||
catch e
|
||||
@@ -1160,7 +1177,7 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function;
|
||||
showerror(io, e)
|
||||
errorMsg = String(take!(io))
|
||||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||||
println("\n~~~ SQLLLM generatequestion() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, " ", @__LINE__)
|
||||
println("\n~~~ SQLLLM generatequestion() Attempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, ":", @__LINE__)
|
||||
noise = GeneralUtils.randstrings(3, 5)
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user