From 200a1d3e2371a11f4184086acf178ed8e41538c8 Mon Sep 17 00:00:00 2001 From: tonaerospace Date: Fri, 14 Mar 2025 12:32:09 +0700 Subject: [PATCH] update --- src/interface.jl | 439 ++------------------------------------------- src/llmfunction.jl | 107 ++++------- test/runtest.jl | 25 ++- 3 files changed, 71 insertions(+), 500 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index c37442d..e6e8355 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -233,6 +233,10 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, response end + # sometime LLM output something like **Comprehension**: which is not expected + response = replace(response, "**"=>"") + response = replace(response, "***"=>"") + # some time LLM output Plan_1: so we need to detect and replace topic numbering regex = r"_[0-1000]+:" matches = collect(eachmatch(regex, response)) @@ -250,9 +254,9 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, dictkey = ["comprehension", "plan", "action_name", "action_input"] # detect if there are more than 1 key per categories - count = GeneralUtils.countGivenWords(response, header) + wordcount = GeneralUtils.countGivenWords(response, header) duplicateKeywordFlag = false - for (i, v) in enumerate(count) + for (i, v) in enumerate(wordcount) keyword = header[i] keywordNumber = v if keywordNumber > 1 @@ -355,420 +359,8 @@ julia> # Signature """ -# function evaluator(state::T1, text2textInstructLLM::Function; -# insertSQLVectorDB::Union{Function, Nothing}=nothing -# ) where {T1<:AbstractDict} - -# # systemmsg = -# # """ -# # You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions. - -# # Definitions: -# # "question" is the user's question. -# # "thought" is step-by-step reasoning about the current situation. -# # "plan" is what to do to complete the task from the current situation. -# # "action" is the taken action which can be one of the following functions: -# # 1) TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. -# # 2) GETDATA[instruction], which you can use to get the data from the database. -# # 3) ANSWERBOX[answer], which returns your answer to the user. "answer" is your answer to the user question. -# # "observation" is result of the action in JSON format. - -# # At each round of conversation, the user will give you: -# # Context: ... -# # Trajectories: ... - -# # You should then respond to the user with: -# # - Original_question: Repeat the original question. -# # - Evaluation (you must evaluate all of the following points): -# # 1) Analyze the trajectories of a solution to answer the user's original question. -# # Given a question and a trajectory, evaluate its correctness and provide your reasoning and -# # analysis in detail. Focus on the latest thought, action, and observation. -# # Incomplete trajectories can be correct if the thoughts and actions so far are correct, -# # even if the answer is not found yet. Do not generate additional thoughts or actions. -# # 2) How the observation addresses the original question? -# # 3) Provide suggestion (if applicable). -# # - Score: Correctness score s where s is an integer from 0 to 10. -# # - Accepted_as_answer: Decide whether to accept the observation as the answer to the original question. -# # 1) The accepted observation should directly answer the question. -# # 2) The possible responses are either 'Yes' or 'No.' - -# # You should only respond in JSON format as described below: -# # {"original_question": ..., "evaluation": ..., "score": ..., "accepted_as_answer": ...} - -# # Here are correct trajectory examples: -# # user: -# # { -# # "question": "I'm looking for a sedan with an automatic driving feature.", -# # "thought_1": "I have many types of sedans in my inventory, each with diverse features.", -# # "thought_2": "I should check our inventory first to see if we have the one our customer wants.", -# # "action_1": {"name": "inventory", "input": "a sedan with an automatic driving feature"}, -# # "observation_1": "Yiem Model A, Conez Model B" -# # } -# # assistant: -# # { -# # "original_question": "the user is looking for a sedan with an automatic driving feature.", -# # "evaluation": "This trajectory is correct because it is logical to use the INVENTORY function to search for inventory based on the details provided in the question, which could lead to a potential answer. The user is asking whether do you have a sedan with an automatic driving feature and the observation provides a list of sedan models that you have. Thus, it is accepted as the answer.", -# # "score": 10, -# # "accepted_as_answer": "Yes" -# # } - -# # user: -# # { -# # "question": "How many cars that fitted with a stereo we have?", -# # "thought_1": "I have many types of car in my inventory, each with diverse features.", -# # "thought_3": "I should check our inventory.", -# # "action_1": {"name": "inventory", "input": "vehicle with a stereo"}, -# # "observation_1": "2015 Conez truck." -# # } -# # assistant: -# # { -# # "evaluation": “This approach is correct. It's reasonable to use the INVENTORY function to search for inventory. However, the query asked for a car but the observation was a truck. Thus it is not accepted as the answer. To improve, make sure to input the correct terms and match the requested criteria accurately.”, -# # "score": 5, -# # "accepted_as_answer": "No" -# # } - -# # Here are incorrect trajectory examples: -# # user: -# # { -# # "question": "I'm looking for a sedan with an automatic driving feature. Do you have it in stock?", -# # "thought_1": "I have many types of sedans in my inventory, each with diverse features.", -# # "thought_2": "I will use SEARCHINTERNET function to search for the car.", -# # "action_1": {"name": "SEARCHINTERNET", "input": "a sedan with an automatic driving feature.}, -# # "observation_1": "Teza Model A, Teza Model B" -# # } -# # assistant: -# # { -# # "evaluation": "This trajectory is incorrect. Using the SEARCHINTERNET function to search for a sedan in the Internet is illogical because the question asked for the cars available for sale at your dealership. To improve, ensure that you read the question clearly.", -# # "score": 0, -# # "accepted_as_answer": "No" -# # } - -# # Let's begin! -# # """ - -# # systemmsg = -# # """ -# # You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions. - -# # Definitions: -# # "question" is the user's question. -# # "thought" is step-by-step reasoning about the current situation. -# # "plan" is what to do to complete the task from the current situation. -# # “action_name” is the name of the action taken, which can be one of the following functions: -# # 1) CHATBOX[text], which you can use to talk with the user. "text" is in verbal English. -# # 2) WINESTOCK[query], which you can use to find info about wine in your inventory. "query" is a search term in verbal English. The best query must includes "budget", "type of wine", "characteristics of wine" and "food pairing". -# # "action_input" is the input to the action -# # "observation" is result of the action. - -# # At each round of conversation, the user will give you: -# # Context: ... -# # Trajectories: ... - -# # You should then respond to the user with: -# # - original_question: Repeat the original question. -# # - evaluation (you must evaluate all of the following points in a single paragraph): -# # 1) Analyze the trajectories of a solution to answer the user's original question. -# # Given a question and a trajectory, evaluate its correctness and provide your reasoning and -# # analysis in detail. Focus on the latest thought, action, and observation. -# # Incomplete trajectories can be correct if the thoughts and actions so far are correct, -# # even if the answer is not found yet. Do not generate additional thoughts or actions. -# # 2) How the observation addresses the question exactly? -# # - accepted_as_answer: Decide whether to accept the observation as the answer to the original question. -# # 1) if the observation's content directly answers the question then just accept it as the answer. Oherwise, it is not. The possible responses are either 'Yes' or 'No.' -# # - score: Correctness score s where s is a single integer between 0 to 9. -# # 1) 0 means the trajectories are incorrect. -# # 2) 9 means the trajectories are correct, and the observation's content directly answers the question. -# # - suggestion: if accepted_as_answer is "No", provide suggestion. - -# # You should only respond in format as described below: -# # original_question: ... -# # evaluation: ... -# # accepted_as_answer: ... -# # score: ... -# # suggestion: ... - -# # Let's begin! -# # """ - -# systemmsg = -# """ -# You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions. - -# Definitions: -# "question" is the user's question -# "understanding" is agent's understanding about the current situation -# "reasoning" is agent's step-by-step reasoning about the current situation -# "plan" is agent's plan to complete the task from the current situation -# "action_name" is the name of the action taken, which can be one of the following functions: -# - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database. -# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. -# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. -# "action_input" is the input to the action -# "observation" is result of the preceding immediate action - -# At each round of conversation, the user will give you: -# Trajectory: ... -# Error note: error note from your previous attempt - -# You must follow the following guidelines: -# - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. - -# You should then respond to the user with: -# 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. -# - Evaluate the correctness of each section and the overall trajectory based on the given question. -# - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation. -# - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. -# - Do not generate additional thoughts or actions. -# 2) Answer_evaluation: -# - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question -# - State your rationale -# 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No" -# Bad example (The observation didn't answers the question): -# question: Find cars with 4 wheels. -# observation: There are an apple in the table. -# Good example (The observation answers the question): -# question: Find cars with a stereo. -# observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black. -# 4) Score: Correctness score s where s is a single integer between 0 to 9. -# Score guideline: -# - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed -# - 4 indicates that the trajectory are correct but the observation is incorrect or failed -# - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. -# - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. -# 5) Suggestion: if accepted_as_answer is "No", provide suggestion. - -# You should only respond in format as described below: -# Trajectory_evaluation: ... -# Answer_evaluation: ... -# Accepted_as_answer: ... -# Score: ... -# Suggestion: ... - -# Let's begin! -# """ - -# thoughthistory = "" -# for (k, v) in state[:thoughtHistory] -# thoughthistory *= "$k: $v\n" -# end - -# errornote = "" - -# for attempt in 1:5 -# usermsg = -# """ -# Trajectory: $thoughthistory -# Error note: $errornote -# """ - -# _prompt = -# [ -# Dict(:name=> "system", :text=> systemmsg), -# Dict(:name=> "user", :text=> usermsg) -# ] - -# # put in model format -# prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen") -# prompt *= -# """ -# <|start_header_id|>assistant<|end_header_id|> -# """ - -# header = ["Trajectory_evaluation", "Answer_evaluation", "Accepted_as_answer", "Score", "Suggestion"] - -# try -# response = text2textInstructLLM(prompt) -# # make sure every header is in the response -# for i in header -# detected = GeneralUtils.detect_keyword(i, response) -# if detected === nothing -# error("Keyword $i not found in response") -# end -# end - -# responsedict = GeneralUtils.textToDict(response, -# header; -# rightmarker=":", symbolkey=true, lowercasekey=true) - -# # check if dict has all required value -# trajectoryevaluation_text::AbstractString = responsedict[:trajectory_evaluation] -# answerevaluation_text::AbstractString = responsedict[:answer_evaluation] -# # responsedict[:score] = replace(responsedict[:score], r"\(.*?\)" => "") # remove (...) if there is any. -# responsedict[:score] = responsedict[:score][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. -# responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5 -# score::Integer = responsedict[:score] -# accepted_as_answer::AbstractString = responsedict[:accepted_as_answer] -# suggestion::AbstractString = responsedict[:suggestion] - -# if accepted_as_answer ∉ ["Yes", "No"] # [PENDING] add errornote into the prompt -# error("generated accepted_as_answer has wrong format") -# end - -# # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension) -# state[:evaluation] = "$(responsedict[:trajectory_evaluation]) $(responsedict[:answer_evaluation])" -# state[:evaluationscore] = responsedict[:score] -# state[:accepted_as_answer] = responsedict[:accepted_as_answer] -# state[:suggestion] = responsedict[:suggestion] - -# # mark as terminal state when the answer is achieved -# if accepted_as_answer == "Yes" - -# # mark the state as terminal state because the evaluation say so. -# state[:isterminal] = true - -# # evaluation score as reward because different answers hold different value for the user. -# state[:reward] = responsedict[:score] -# end -# println("\n~~~ Evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# pprintln(Dict(responsedict)) - -# return responsedict[:score] -# catch e -# io = IOBuffer() -# showerror(io, e) -# errorMsg = String(take!(io)) -# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) -# println("") -# println("Attempt $attempt. Error occurred: $errorMsg\n$st") -# println("") -# end -# end -# error("evaluator failed to generate an evaluation") -# end -function evaluator(state::T1, text2textInstructLLM::Function; - insertSQLVectorDB::Union{Function, Nothing}=nothing +function evaluator(state::T1, text2textInstructLLM::Function ) where {T1<:AbstractDict} - - # systemmsg = - # """ - # You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions. - - # Definitions: - # "question" is the user's question. - # "thought" is step-by-step reasoning about the current situation. - # "plan" is what to do to complete the task from the current situation. - # "action" is the taken action which can be one of the following functions: - # 1) TABLEINFO[list_of_table_name], which you can use to get the data type of a table column. - # 2) GETDATA[instruction], which you can use to get the data from the database. - # 3) ANSWERBOX[answer], which returns your answer to the user. "answer" is your answer to the user question. - # "observation" is result of the action in JSON format. - - # At each round of conversation, the user will give you: - # Context: ... - # Trajectories: ... - - # You should then respond to the user with: - # - Original_question: Repeat the original question. - # - Evaluation (you must evaluate all of the following points): - # 1) Analyze the trajectories of a solution to answer the user's original question. - # Given a question and a trajectory, evaluate its correctness and provide your reasoning and - # analysis in detail. Focus on the latest thought, action, and observation. - # Incomplete trajectories can be correct if the thoughts and actions so far are correct, - # even if the answer is not found yet. Do not generate additional thoughts or actions. - # 2) How the observation addresses the original question? - # 3) Provide suggestion (if applicable). - # - Score: Correctness score s where s is an integer from 0 to 10. - # - Accepted_as_answer: Decide whether to accept the observation as the answer to the original question. - # 1) The accepted observation should directly answer the question. - # 2) The possible responses are either 'Yes' or 'No.' - - # You should only respond in JSON format as described below: - # {"original_question": ..., "evaluation": ..., "score": ..., "accepted_as_answer": ...} - - # Here are correct trajectory examples: - # user: - # { - # "question": "I'm looking for a sedan with an automatic driving feature.", - # "thought_1": "I have many types of sedans in my inventory, each with diverse features.", - # "thought_2": "I should check our inventory first to see if we have the one our customer wants.", - # "action_1": {"name": "inventory", "input": "a sedan with an automatic driving feature"}, - # "observation_1": "Yiem Model A, Conez Model B" - # } - # assistant: - # { - # "original_question": "the user is looking for a sedan with an automatic driving feature.", - # "evaluation": "This trajectory is correct because it is logical to use the INVENTORY function to search for inventory based on the details provided in the question, which could lead to a potential answer. The user is asking whether do you have a sedan with an automatic driving feature and the observation provides a list of sedan models that you have. Thus, it is accepted as the answer.", - # "score": 10, - # "accepted_as_answer": "Yes" - # } - - # user: - # { - # "question": "How many cars that fitted with a stereo we have?", - # "thought_1": "I have many types of car in my inventory, each with diverse features.", - # "thought_3": "I should check our inventory.", - # "action_1": {"name": "inventory", "input": "vehicle with a stereo"}, - # "observation_1": "2015 Conez truck." - # } - # assistant: - # { - # "evaluation": “This approach is correct. It's reasonable to use the INVENTORY function to search for inventory. However, the query asked for a car but the observation was a truck. Thus it is not accepted as the answer. To improve, make sure to input the correct terms and match the requested criteria accurately.”, - # "score": 5, - # "accepted_as_answer": "No" - # } - - # Here are incorrect trajectory examples: - # user: - # { - # "question": "I'm looking for a sedan with an automatic driving feature. Do you have it in stock?", - # "thought_1": "I have many types of sedans in my inventory, each with diverse features.", - # "thought_2": "I will use SEARCHINTERNET function to search for the car.", - # "action_1": {"name": "SEARCHINTERNET", "input": "a sedan with an automatic driving feature.}, - # "observation_1": "Teza Model A, Teza Model B" - # } - # assistant: - # { - # "evaluation": "This trajectory is incorrect. Using the SEARCHINTERNET function to search for a sedan in the Internet is illogical because the question asked for the cars available for sale at your dealership. To improve, ensure that you read the question clearly.", - # "score": 0, - # "accepted_as_answer": "No" - # } - - # Let's begin! - # """ - - # systemmsg = - # """ - # You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions. - - # Definitions: - # "question" is the user's question. - # "thought" is step-by-step reasoning about the current situation. - # "plan" is what to do to complete the task from the current situation. - # “action_name” is the name of the action taken, which can be one of the following functions: - # 1) CHATBOX[text], which you can use to talk with the user. "text" is in verbal English. - # 2) WINESTOCK[query], which you can use to find info about wine in your inventory. "query" is a search term in verbal English. The best query must includes "budget", "type of wine", "characteristics of wine" and "food pairing". - # "action_input" is the input to the action - # "observation" is result of the action. - - # At each round of conversation, the user will give you: - # Context: ... - # Trajectories: ... - - # You should then respond to the user with: - # - original_question: Repeat the original question. - # - evaluation (you must evaluate all of the following points in a single paragraph): - # 1) Analyze the trajectories of a solution to answer the user's original question. - # Given a question and a trajectory, evaluate its correctness and provide your reasoning and - # analysis in detail. Focus on the latest thought, action, and observation. - # Incomplete trajectories can be correct if the thoughts and actions so far are correct, - # even if the answer is not found yet. Do not generate additional thoughts or actions. - # 2) How the observation addresses the question exactly? - # - accepted_as_answer: Decide whether to accept the observation as the answer to the original question. - # 1) if the observation's content directly answers the question then just accept it as the answer. Oherwise, it is not. The possible responses are either 'Yes' or 'No.' - # - score: Correctness score s where s is a single integer between 0 to 9. - # 1) 0 means the trajectories are incorrect. - # 2) 9 means the trajectories are correct, and the observation's content directly answers the question. - # - suggestion: if accepted_as_answer is "No", provide suggestion. - - # You should only respond in format as described below: - # original_question: ... - # evaluation: ... - # accepted_as_answer: ... - # score: ... - # suggestion: ... - - # Let's begin! - # """ systemmsg = """ @@ -861,6 +453,10 @@ function evaluator(state::T1, text2textInstructLLM::Function; response = text2textInstructLLM(prompt) + # sometime LLM output something like **Comprehension**: which is not expected + response = replace(response, "**"=>"") + response = replace(response, "***"=>"") + # make sure every header is in the response for i in header detected = GeneralUtils.detect_keyword(i, response) @@ -1127,8 +723,7 @@ function transition(state::T, args::NamedTuple elseif thoughtDict[:action_name] == "GETDATA" response = SQLexecution(executeSQL, thoughtDict[:action_input]) if response[:success] - # intention = Dict(:intention=> "$(thoughtDict[:plan])") - extracted = extractContent_dataframe(response[:result], text2textInstructLLM) + extracted = extractContent_dataframe(response[:result], text2textInstructLLM, thoughtDict[:action_input]) (rawresponse=response[:result], result=extracted, errormsg=nothing, success=true) else (result=nothing, errormsg=response[:errormsg], success=false) @@ -1144,8 +739,7 @@ function transition(state::T, args::NamedTuple reward::Integer = haskey(response, :reward) ? response[:reward] : 0 isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false newNodeKey, newstate = makeNewState(state, thoughtDict, rawresponse, JSON3.write(result), select, reward, isterminal) - progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM; - insertSQLVectorDB=insertSQLVectorDB) + progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM) return (newNodeKey=newNodeKey, newstate=newstate, progressvalue=progressvalue) end @@ -1239,7 +833,7 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; response = SQLexecution(executeSQL, sql) if response[:success] # intention = Dict(:intention=> "$(thoughtDict[:plan])") - extracted = extractContent_dataframe(response[:result], text2textInstructLLM) + extracted = extractContent_dataframe(response[:result], text2textInstructLLM, sql) return (text=extracted, rawresponse=response[:result]) end end @@ -1278,13 +872,14 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; earlystop(state) = state[:reward] >= 8 ? true : false _, _, resultState = LLMMCTS.runMCTS(initialstate, transition, transitionargs; - horizontalSampleExpansionPhase=3, + horizontalSampleExpansionPhase=2, horizontalSampleSimulationPhase=2, maxSimulationDepth=10, maxiterations=1, explorationweight=1.0, earlystop=earlystop, - saveSimulatedNode=true) + saveSimulatedNode=true, + multithread=true) latestKey, latestInd = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation") action_input = Symbol("action_input_$latestInd") # latest sql sql = resultState[:thoughtHistory][action_input] diff --git a/src/llmfunction.jl b/src/llmfunction.jl index c0e21d1..7688c86 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -352,76 +352,39 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM: Hints = "None" - # """ - # Here are some useful SQL programs: - # $usefulSQL - # """ + systemmsg = + """ + You are an assistant helping the user to execute SQL code from the user's query. - # systemmsg = - # """ - # You are an assistant helping the user to execute SQL code from the user's query. + At each round of conversation, the user will give you: + Context: ... + User intention: ... + Code executed from the last round: ... + Execution error: execution error of the last round code. - # At each round of conversation, the user will give you: - # Context: ... - # User intention: ... - # Code executed from the last round: ... - # Execution error: execution error of the last round code. + You should consider the following guidelines: + - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search. - # You should consider the following guidelines: - # - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search. + You should then respond to the user with: + 1) Comprehension: + - State your comprehension about the current situation. + 3) Plan: Step-by-step instructions of how to complete the task. + - Focus on improving the code from the last round. + - Do not create any table in the database. + 4) Code: + - Write new improved code. + - Do not wrap the code and no comment as it will be executed directly without any modification against the database. - # You should then respond to the user with: - # - thought: Why the code does not complete the task. What does the execution error imply exactly? - # - plan: Step-by-step instructions of how to complete the task. - # 1) Focus on improving the code from the last round. - # 2) Do not create any table in the database. - # - code: - # 1) Write new improved code. - # 2) Do not wrap the code and no comment as it will be executed directly without any modification against the database. + You should only respond in format as described below and nothing more: + Comprehension: ... + Plan: + 1) ... + 2) ... + ... + Code: ... - # You should only respond in format as described below and nothing more: - # thought: ... - # plan: - # 1) ... - # 2) ... - # ... - # code: ... - - # Let's begin! - # """ - - systemmsg = """ - You are an assistant helping the user to execute SQL code from the user's query. - - At each round of conversation, the user will give you: - Context: ... - User intention: ... - Code executed from the last round: ... - Execution error: execution error of the last round code. - - You should consider the following guidelines: - - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search. - - You should then respond to the user with: - 1) Comprehension: - - State your comprehension about the current situation. - 3) Plan: Step-by-step instructions of how to complete the task. - - Focus on improving the code from the last round. - - Do not create any table in the database. - 4) Code: - - Write new improved code. - - Do not wrap the code and no comment as it will be executed directly without any modification against the database. - - You should only respond in format as described below and nothing more: - Comprehension: ... - Plan: - 1) ... - 2) ... - ... - Code: ... - - Let's begin! - """ + Let's begin! + """ noise = "" note_flag = "" @@ -597,7 +560,7 @@ end # Signature """ -function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function +function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function, action::String )::String tablesize = size(df) row = tablesize[1] @@ -633,8 +596,7 @@ function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function You are an assistant that readouts the resulting table after the user executing SQL command. At each round of conversation, the user will give you: - - User intention: ... - - Resulting table dimension: ... + - User SQL: the SQL query user executed. - Resulting table: The resulting table after executing the user's intention. You should then respond to the user with: @@ -652,9 +614,12 @@ function extractContent_dataframe(df::DataFrame, text2textInstructLLM::Function Let's begin! """ - usermsg = """ - Resulting table: $dfstr - """ + + usermsg = + """ + User SQL: $action + Resulting table: $dfstr + """ _prompt = [ Dict(:name => "system", :text => systemmsg), diff --git a/test/runtest.jl b/test/runtest.jl index 8240c3d..23847e3 100644 --- a/test/runtest.jl +++ b/test/runtest.jl @@ -29,7 +29,7 @@ function executeSQLVectorDB(sql) return result end -function text2textInstructLLM(prompt::String) +function text2textInstructLLM(prompt::String; maxattempt=3) msgMeta = GeneralUtils.generate_msgMeta( config[:externalservice][:loadbalancer][:mqtttopic]; msgPurpose="inference", @@ -51,8 +51,19 @@ function text2textInstructLLM(prompt::String) ) ) - _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120, maxattempt=2) - response = _response[:response][:text] + response = nothing #[WORKING] receive nothing when LLM container reset + for attempts in 1:maxattempt + _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120, maxattempt=2) + response = _response[:response][:text] + if response !== nothing + break + else + println("\n attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + pprintln(outgoingMsg) + println(" attempt $attempts/$maxattempt failed ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") + sleep(3) + end + end return response end @@ -103,8 +114,8 @@ function similarSQLVectorDB(query; maxdistance::Integer=100) df = findSimilarTextFromVectorDB(query, tablename, "function_input_embedding", executeSQLVectorDB) row, col = size(df) - # distance = row == 0 ? Inf : df[1, :distance] - distance = 100 # CHANGE this is for testing only + distance = row == 0 ? Inf : df[1, :distance] + # distance = 100 # CHANGE this is for testing only if row != 0 && distance < maxdistance # if there is usable SQL, return it. output_b64 = df[1, :function_output_base64] # pick the closest match @@ -146,8 +157,8 @@ sessionId = "555" # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?") -# query = "How many wines are from United States?" -query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40" +query = "How many wines are from Italy?" +# query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40" # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza" # query = "wine_type: white, country: Austria, food to be served with wine: pork" # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza"