diff --git a/src/interface.jl b/src/interface.jl index a16f05d..27ec663 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -136,18 +136,13 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function systemmsg = """ You are a helpful assistant that find the data from a database to satisfy the user's question. - You are also eager to improve your helpfulness. + You are working under your mentor supervision and you are also eager to improve your helpfulness. For your information: - Observation: Result of the immediately preceding action At each round of conversation, you will be given the following information: - table_schema: schema of tables in the database - most_relevant_SQL: The closest known SQL for this question - query_result_of_most_relevant_SQL: the query result when executing the most_relevant_SQL against a database. - progress: your progress so far - evaluation: Evaluation of the immediately preceding action and observation - suggestion: Suggestion for the immediately preceding action and observation + context: additional information about the current situation You must follow the following guidelines: - Keep SQL queries focused only on the provided information. @@ -160,22 +155,24 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function - If there is no search result from the database, remove the restrictive criteria until a search result is available, and proceed from there. You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input: - Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. - Action_name: (Typically corresponds to the execution of the first step in your plan) + 1) plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. + 2) action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following function names: - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. - 4) Action_input: Input to the action + 3) action_input: Input to the action - You should only respond in format as described below: - Plan: ... - Action_name: ... - Action_input: ... + You should only respond in JSON format as described below: + { + "plan": "...", + "action_name": "...", + "action_input": "..." + } Let's begin! """ - + requiredKeys = [:plan, :action_name, :action_input] workprogress = "" for (k, v) in state[:thoughtHistory] if k ∉ [:question] @@ -193,13 +190,6 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function similarSQL_ = sql !== nothing ? sql : "None" end - header = ["Plan:", "Action_name:", "Action_input:"] - dictkey = ["plan", "action_name", "action_input"] - - llmkwargs=Dict( - :num_ctx => 32768, - :temperature => 0.5, - ) for attempt in 1:maxattempt @@ -208,22 +198,19 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function context = """ - + This is schema of tables in the database: $(additionalinfo[:tablelist]) - + The closest known SQL for this question is: $similarSQL_ - + This is the query result when executing the most_relevant_SQL against a database. You can use this to see how the data are stored. winery: Chateau Montelena, wine_name: The Montelena Estate Cabernet Sauvignon, wine_id: 97264f71-007c-4cce-a3fe-2cc88fba4d05, vintage: 2017, region: Napa Valley, country: United States, wine_type: red, grape: Cabernet Sauvignon, serving_temperature: 15 to 18 Celsius, sweetness: 1, intensity: 5, tannin: 4, acidity: 4, tasting_notes: oak, vanilla, tobacco, blackberry, plum, black cherry, leather, earthy, smoke, price: 19.95, currency: USD - + your work progress so far: $workprogress - - $(state[:evaluation]) - - + This is your mentor's suggestion for the immediately preceding action and observation $(state[:suggestion]) P.S. $errornote @@ -240,72 +227,40 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function prompt = GeneralUtils.formatLLMtext(unformatPrompt, llmFormatName) # add info prompt = prompt * context - response = text2textInstructLLM(prompt; llmkwargs=llmkwargs) + response = text2textInstructLLM(prompt) response = GeneralUtils.deFormatLLMtext(response, llmFormatName) think, response = GeneralUtils.extractthink(response) - - - # LLM tends to generate observation given that it is in the input - response = - if occursin("observation:", response) - string(split(response, "observation:")[1]) - elseif occursin("Observation:", response) - string(split(response, "Observation:")[1]) - elseif occursin("observation_", response) - string(split(response, "observation_")[1]) - elseif occursin("Observation_", response) - string(split(response, "Observation_")[1]) - else - response - end - - # sometime LLM output something like **Comprehension**: which is not expected - response = replace(response, "**"=>"") - response = replace(response, "***"=>"") - - # # some time LLM output Plan_1: so we need to detect and replace topic numbering - # regex = r"_[0-1000]+:" - # matches = collect(eachmatch(regex, response)) - # for m in matches - # response = replace(response, string(m.match)=>":") - # end - if occursin("NULL", response) errornote = "\nYour previous attempt was NULL. This is not allowed" println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - # # detect if there are more than 1 key per categories - # wordcount = GeneralUtils.countGivenWords(response, header) - # duplicateKeywordFlag = false - # for (i, v) in enumerate(wordcount) - # keyword = header[i] - # keywordNumber = v - # if keywordNumber > 1 - # errornote = "\nSQL query has duplicated keyword, $keyword" - # println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # duplicateKeywordFlag = true - # break - # end - # end - # duplicateKeywordFlag == true ? continue : nothing - - # check whether response has all header - detected_kw = GeneralUtils.detect_keyword(header, response) - if 0 ∈ values(detected_kw) - errornote = "\nYour previous attempt did not have all points according to the required response format" - println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "\nYour previous attempt has duplicated points according to the required response format" - println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + responsedict = nothing + try + responsedict = copy(JSON3.read(response)) + catch + println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) + # check whether all answer's key points are in responsedict + _responsedictKey = keys(responsedict) + responsedictKey = [i for i in _responsedictKey] # convert into a list + is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] + + if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) + errornote = "Your previous attempt has more key points than answer's required key points." + println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + elseif !all(is_requiredKeys_in_responsedictKey) + zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) + missingkeys = [requiredKeys[i] for i in zeroind] + errornote = "$missingkeys are missing from your previous response" + println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + end delete!(responsedict, :observation) @@ -336,13 +291,13 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function end end - for i ∈ Symbol.(dictkey) - if length(JSON3.write(responsedict[i])) == 0 - errornote = "Your previous attempt has empty value for $i" - println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - end + # for i ∈ Symbol.(dictkey) + # if length(JSON3.write(responsedict[i])) == 0 + # errornote = "Your previous attempt has empty value for $i" + # println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end + # end println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") pprintln(Dict(responsedict)) @@ -634,7 +589,7 @@ julia> # Signature """ -function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, llmFormatName::String; +function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFormatName::String; maxattempt=10 ) where {T1<:AbstractDict} @@ -653,8 +608,8 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll "observation" is result of the preceding immediate action At each round of conversation, you will be given the following information: - Trajectory: A history of how you worked on the question chronologically. - table_schema: schema of tables in the database + trajectory: A history of how you worked on the question chronologically + evaluatee_context: The context that evaluatee use to make a decision You must follow the following guidelines: - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. @@ -704,18 +659,20 @@ function evaluator(state::T1, additionalinfo, text2textInstructLLM::Function, ll errornote = "N/A" for attempt in 1:maxattempt - usermsg = - """ - Trajectory: $thoughthistory - """ + usermsg = + """ + + $thoughthistory + + """ context = - """ - - - $(additionalinfo[:tablelist]) - - P.S. $errornote - + """ + + + thoughtDict[:context] + + P.S. $errornote + """ unformatPrompt = @@ -1056,7 +1013,7 @@ function transition(state::T, args::NamedTuple reward::Integer = haskey(response, :reward) ? response[:reward] : 0 isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON3.write(result), select, reward, isterminal) - progressvalue::Integer = evaluatorF(newstate, context, text2textInstructLLM, llmFormatName) + progressvalue::Integer = evaluatorF(newstate, thoughtDict, text2textInstructLLM, llmFormatName) return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue) end @@ -1333,12 +1290,12 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; # JSON3.pretty(io, highValueState) # end selected = compareState(query, highValueState, text2textInstructLLM, llmFormatName) - resultState = highValueState[selected] #BUG compareState() select 0 + resultState = highValueState[selected] end latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation") action_input = Symbol("action_input_$latestInd") # latest sql sql = resultState[:thoughtHistory][action_input] - extracted = resultState[:thoughtHistory][latestKey] + extractedTableContent = resultState[:thoughtHistory][latestKey] # add to vectorDB only if the answer is achieved and the state is terminal if insertSQLVectorDB !== nothing && resultState[:isterminal] == true && @@ -1347,11 +1304,11 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; insertSQLVectorDB(resultState[:thoughtHistory][:question], sql) end - if extracted === nothing + if extractedTableContent === nothing println("\nSQLLLM query() return nothing ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - result = (text=extracted, rawresponse=resultState[:rawresponse]) + result = (text=extractedTableContent, rawresponse=resultState[:rawresponse]) return result end