From 568e0ff54f73cd56a1bea8c4a0b13da1aca42957 Mon Sep 17 00:00:00 2001 From: tonaerospace Date: Thu, 20 Mar 2025 16:08:40 +0700 Subject: [PATCH] update --- src/interface.jl | 316 ++++--------------------------- test/Manifest.toml | 41 ++++ test/Project.toml | 2 + test/{runtest.jl => runtests.jl} | 0 4 files changed, 80 insertions(+), 279 deletions(-) create mode 100644 test/Manifest.toml create mode 100644 test/Project.toml rename test/{runtest.jl => runtests.jl} (100%) diff --git a/src/interface.jl b/src/interface.jl index f3c9244..66f7e32 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -159,11 +159,11 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query". - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. - You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action: - 1) Comprehension: - - State your comprehension about the current situation. - 2) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. - 3) Action_name (Must be aligned with your plan): Can be one of the following functions: + You should then respond to the user with interleaving Comprehension, Plan, Action_name, Action_input: + Comprehension: state your comprehension about the current situation. + Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. + Action_name: (Typically corresponds to the execution of the first step in your plan) + Can be one of the following function names: - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. @@ -195,6 +195,8 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, similarSQL_ = sql !== nothing ? sql : "None" end + header = ["Comprehension:", "Plan:", "Action_name:", "Action_input:"] + dictkey = ["comprehension", "plan", "action_name", "action_input"] for attempt in 1:10 QandA = generatequestion(state, context, text2textInstructLLM; similarSQL=similarSQL_) @@ -252,37 +254,31 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, continue end - header = ["Comprehension:", "Plan:", "Action_name:", "Action_input:"] - dictkey = ["comprehension", "plan", "action_name", "action_input"] - - # detect if there are more than 1 key per categories - wordcount = GeneralUtils.countGivenWords(response, header) - duplicateKeywordFlag = false - for (i, v) in enumerate(wordcount) - keyword = header[i] - keywordNumber = v - if keywordNumber > 1 - errornote = "\nSQL query has duplicated keyword, $keyword" - println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - duplicateKeywordFlag = true - break - end - end - duplicateKeywordFlag == true ? continue : nothing + # # detect if there are more than 1 key per categories + # wordcount = GeneralUtils.countGivenWords(response, header) + # duplicateKeywordFlag = false + # for (i, v) in enumerate(wordcount) + # keyword = header[i] + # keywordNumber = v + # if keywordNumber > 1 + # errornote = "\nSQL query has duplicated keyword, $keyword" + # println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # duplicateKeywordFlag = true + # break + # end + # end + # duplicateKeywordFlag == true ? continue : nothing # check whether response has all header - kw = [] - # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list - for keyword in header - detected = GeneralUtils.detect_keyword(keyword, response) - push!(kw, detected) - end - if nothing ∈ kw - println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue # try again next loop + detected_kw = GeneralUtils.detect_keyword(header, response) + if sum(values(detected_kw)) < length(header) + errornote = "\nSQL decisionMaker() response does not have all header" + continue + elseif sum(values(detected_kw)) > length(header) + errornote = "\nSQL decisionMaker() response has duplicated header" + continue end - # textToDict() search for action_input responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=true) @@ -315,7 +311,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, end end - for i ∈ [:comprehension, :plan, :action_name, :action_input] + for i ∈ Symbol.(dictkey) if length(JSON3.write(responsedict[i])) == 0 errornote = "\n $i is empty" println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") @@ -323,14 +319,14 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, end end - # check if there are more than 1 key per categories - for i ∈ [:comprehension, :plan, :action_name, :action_input] - matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i) - if length(matchkeys) > 1 - errornote = "\n $i has more than one key" - println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end + # check whether response has all header + detected_kw = GeneralUtils.detect_keyword(header, response) + if sum(values(detected_kw)) < length(header) + errornote = "\nSQL decisionMaker() response does not have all header" + continue + elseif sum(values(detected_kw)) > length(header) + errornote = "\nSQL decisionMaker() response has duplicated header" + continue end state[:decisionMaker] = responsedict @@ -340,244 +336,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, end error("DecisionMaker failed to generate a thought \n", response) end -# function decisionMaker(state::T1, context, text2textInstructLLM::Function, -# ; querySQLVectorDBF::Union{T2, Nothing}=nothing -# )::Dict{Symbol, Any} where {T1<:AbstractDict, T2<:Function} -# # lessonDict = -# # if isfile("lesson.json") -# # lessonDict = copy(JSON3.read("lesson.json")) -# # else -# # lessonDict = nothing -# # end - -# # lessonDict = nothing - -# # lesson = -# # if lessonDict === nothing -# # "" -# # else -# # """ -# # You have attempted to help the user before and failed, either because your reasoning for the -# # recommendation was incorrect or your response did not exactly match the user expectation. -# # The following lesson(s) give a plan to avoid failing to help the user in the same way you -# # did previously. Use them to improve your strategy to help the user. - -# # Here are some lessons in JSON format: -# # $(JSON3.write(lessonDict)) - -# # When providing the thought and action for the current trial, that into account these failed -# # trajectories and make sure not to repeat the same mistakes and incorrect answers. -# # """ -# # end - -# systemmsg = -# """ -# You are a helpful assistant that find the data from a database to satisfy the user's query. -# You are also eager to improve your helpfulness. - -# For your information: -# - Observation: Result of the immediately preceding action - -# At each round of conversation, the user will give you the current situation: -# User Query: ... -# Example: ... -# Your Q&A: ... -# Your work progress: ... -# Evaluation: Evaluation of the immediately preceding action and observation -# Suggestion: Suggestion for the immediately preceding action and observation - -# You must follow the following guidelines: -# - Keep SQL queries focused only on the provided information. - -# You should follow the following guidelines: -# - Do not create any table in the database -# - A junction table can be used to link tables together. Another use case is for filtering data. -# - If you can't find a single table that can be used to answer the user's query, try joining multiple tables to see if you can obtain the answer. -# - If you are unable to find the requested information, kindly inform the user, "The current data in our database does not provide the specific answer to your query". -# - Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search. - -# You should then respond to the user with interleaving Understanding, Reasoning, Plan, Action: -# 1) Comprehension: -# - State your comprehension about the current situation. -# 2) Plan: Given the current circumstances, outline a detailed, step-by-step plan to accomplish the task. Be specific. -# 3) Action_name (Must be aligned with your plan): Can be one of the following functions: -# - GETDATA, which you can use to get the data from the database. Action_input for this function must be a single SQL query to be executed against the database. -# For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. -# Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. -# 4) Action_input: Input to the action - -# You should only respond in format as described below: -# Comprehension: ... -# Plan: ... -# Action_name: ... -# Action_input: ... - -# Let's begin! -# """ - -# workprogress = "" -# for (k, v) in state[:thoughtHistory] -# if k ∉ [:question] -# workprogress *= "$k: $v\n" -# end -# end - -# response = nothing # store for show when error msg show up -# errornote = "" - -# # provide similar sql only for the first attempt -# similarSQL_ = "None" -# if length(state[:thoughtHistory]) == 1 -# sql, distance = querySQLVectorDBF(state[:thoughtHistory][:question]) -# similarSQL_ = sql !== nothing ? sql : "None" -# end - - -# for attempt in 1:10 -# QandA = generatequestion(state, context, text2textInstructLLM; similarSQL=similarSQL_) - -# usermsg = -# """ -# $(context[:tablelist]) -# User query: $(state[:thoughtHistory][:question]) -# Example: $similarSQL_ -# Your Q&A: $QandA -# Your work progress: $workprogress -# Evaluation: $(state[:evaluation]) -# Suggestion: $(state[:suggestion]) -# $errornote -# """ - -# _prompt = -# [ -# Dict(:name=> "system", :text=> systemmsg), -# Dict(:name=> "user", :text=> usermsg) -# ] - -# # put in model format -# prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen") -# response = text2textInstructLLM(prompt) - -# # LLM tends to generate observation given that it is in the input -# response = -# if occursin("observation:", response) -# string(split(response, "observation:")[1]) -# elseif occursin("Observation:", response) -# string(split(response, "Observation:")[1]) -# elseif occursin("observation_", response) -# string(split(response, "observation_")[1]) -# elseif occursin("Observation_", response) -# string(split(response, "Observation_")[1]) -# else -# response -# end - -# # sometime LLM output something like **Comprehension**: which is not expected -# response = replace(response, "**"=>"") -# response = replace(response, "***"=>"") - -# # some time LLM output Plan_1: so we need to detect and replace topic numbering -# regex = r"_[0-1000]+:" -# matches = collect(eachmatch(regex, response)) -# for m in matches -# response = replace(response, string(m.match)=>":") -# end - -# if occursin("NULL", response) -# errornote = "\nSQL decisionMaker() NULL response is not allowed" -# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# header = ["Comprehension:", "Plan:", "Action_name:", "Action_input:"] -# dictkey = ["comprehension", "plan", "action_name", "action_input"] - -# # detect if there are more than 1 key per categories -# wordcount = GeneralUtils.countGivenWords(response, header) -# duplicateKeywordFlag = false -# for (i, v) in enumerate(wordcount) -# keyword = header[i] -# keywordNumber = v -# if keywordNumber > 1 -# errornote = "\nSQL query has duplicated keyword, $keyword" -# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# duplicateKeywordFlag = true -# break -# end -# end -# duplicateKeywordFlag == true ? continue : nothing - -# # check whether response has all header -# kw = [] -# # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list -# for keyword in header -# detected = GeneralUtils.detect_keyword(keyword, response) -# push!(kw, detected) -# end -# if nothing ∈ kw -# println("Some keywords are missing, Required keywords=$header, Response keywords=$kw ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue # try again next loop -# end - -# # textToDict() search for action_input -# responsedict = GeneralUtils.textToDict(response, header; -# dictKey=dictkey, symbolkey=true) - -# delete!(responsedict, :observation) - -# # remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String}) -# if occursin("```", responsedict[:action_input]) -# sql = GeneralUtils.extract_triple_backtick_text(responsedict[:action_input])[1] -# if sql[1:4] == "sql\n" -# sql = sql[5:end] -# end -# sql = split(sql, ';') # some time there are comments in the sql -# sql = sql[1] * ';' - -# responsedict[:action_input] = sql -# end - -# toollist = ["TABLEINFO", "GETDATA"] -# if responsedict[:action_name] ∉ toollist -# errornote = "\nYou must only use the given functions" -# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end - -# for i in toollist -# if occursin(i, responsedict[:action_input]) -# errornote = "\n action_name is in action_input which is not allowed." -# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end -# end - -# for i ∈ [:comprehension, :plan, :action_name, :action_input] -# if length(JSON3.write(responsedict[i])) == 0 -# errornote = "\n $i is empty" -# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end -# end - -# # check if there are more than 1 key per categories -# for i ∈ [:comprehension, :plan, :action_name, :action_input] -# matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i) -# if length(matchkeys) > 1 -# errornote = "\n $i has more than one key" -# println("Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") -# continue -# end -# end - -# state[:decisionMaker] = responsedict - -# return responsedict - -# end -# error("DecisionMaker failed to generate a thought \n", response) -# end """ Assigns a scalar value to each new child node to be used for selec- tion and backpropagation. This value effectively quantifies the agent's progress in task completion, @@ -1429,7 +1188,6 @@ function generatequestion(state::T1, context, text2textInstructLLM::Function; header = ["Understanding:", "Q1:"] dictkey = ["understanding", "q1"] - responsedict = GeneralUtils.textToDict(response, header; dictKey=dictkey, symbolkey=true) response = "Q1: " * responsedict[:q1] diff --git a/test/Manifest.toml b/test/Manifest.toml new file mode 100644 index 0000000..83f035b --- /dev/null +++ b/test/Manifest.toml @@ -0,0 +1,41 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.4" +manifest_format = "2.0" +project_hash = "71d91126b5a1fb1020e1098d9d492de2a4438fd2" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 0000000..0c36332 --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,2 @@ +[deps] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/runtest.jl b/test/runtests.jl similarity index 100% rename from test/runtest.jl rename to test/runtests.jl