From d0c26e52e8f6a17ccaa1cb1f90e5237044680a33 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Wed, 14 May 2025 21:21:35 +0700 Subject: [PATCH] update --- src/interface.jl | 888 ++++++++++++++++++++++++++------------------- src/llmfunction.jl | 191 ++++++---- src/type.jl | 3 +- src/util.jl | 42 ++- 4 files changed, 659 insertions(+), 465 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 18fc8db..b404e08 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -125,9 +125,11 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 # """ # end - recent_ind = GeneralUtils.recentElementsIndex(length(a.memory[:events]), recent; includelatest=true) + recent_ind = GeneralUtils.recentElementsIndex(length(a.chathistory), recent; includelatest=true) recentevents = a.memory[:events][recent_ind] - recentEventsDict = createEventsLog(recentevents; eventindex=recent_ind) + recentchat = createChatLog(a.chathistory[recent_ind]; index=recent_ind) + # recentEventsDict = createEventsLog(recentevents; index=recent_ind) + timeline = createTimeline(recentevents; eventindex=recent_ind) @@ -161,10 +163,6 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 responsedict = similarDecision return responsedict else - - header = ["Plan:", "Action_name:", "Action_input:"] - dictkey = ["plan", "action_name", "action_input"] - # context = # may b add wine name instead of the hold wine data is better # if length(a.memory[:shortmem][:available_wine]) != 0 # winenames = [] @@ -178,83 +176,85 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 # "" # end + systemmsg = + """ + Your name is $(a.name). You are a sommelier for website-based $(a.retailername)'s wine store. You are working under your mentor supervision. + Your goal includes: + 1) Establish a connection with the customer by greeting them warmly + 2) Guide them to select the best wines only from your store's inventory that align with their preferences. + + Your responsibility includes: + 1) According to the store's policy and guidelines, make an informed decision about what you need to do to achieve the goal + 2) Value your mentor's suggestions. + + Your responsibility does NOT includes: + 1) Requesting the user to place an order, make a purchase, or confirm the order. These are the job of our sales team at the store. + 2) Processing sales orders or engaging in any other sales-related activities. These are the job of our sales team at the store. + 3) Answering questions or offering additional services beyond those related to your store's wine recommendations such as discounts, quantity, rewards programs, promotions, delivery options, shipping, boxes, gift wrapping, packaging, personalized messages or something similar. These are the job of our sales team at the store. + + + - Generally speaking, the store inventory has some wines from France, the United States, Australia, Spain, and Italy, but you won't know exactly until you check your inventory. + - If you found wines in the store's database, they are in stock + - Approach each customer with open-ended questions to understand their preferences, budget, and occasion. Once you have these information, you can check your inventory. + - Do not ask the user about wine's flavor e.g. floral, citrusy, nutty or some thing similar as these terms cannot be used to search the database. Other wine characteristics useful for CHECKINVENTORY tools are allowed. + - Once the user has selected their wine, ask the user if they need any further assistance. Do not offer any additional services. If the user doesn't need any further assistance, say goodbye and invite them to come back next time. + - Spicy foods should not be paired with medium and full-bodied red wines. + - We do not sell organic, sustainability and sulfite wine. + + + - Encourage the customer to explore different options and try new things. + - If you are unable to locate the desired item in the database after multiple attempts, it may not be available in your inventory. In such cases, inform the user that the item is unavailable and suggest an alternative instead. + - Your store carries only wine. + - Vintage 0 means non-vintage. + + At each round of conversation, you will be given the following information: + Database search result: the result of a database search using SQL commands you have found so far + + You should then respond to the user with interleaving Plan, Action_name, Action_input: + 1) plan: Based on the current situation, state a complete action plan to complete the task. Be specific. + 2) action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the available tool name + 3) action_input: The input to the action you are about to perform according to your plan. + + You should only respond in JSON format as described below: + { + "plan": "...", + "action_name": "...", + "action_input": "..." + } + + Let's begin! + """ + requiredKeys = [:plan, :action_name, :action_input] + database_search_result = + if length(a.memory[:shortmem][:db_search_result]) != 0 + availableWineToText(a.memory[:shortmem][:db_search_result]) + else + "N/A" + end + errornote = "N/A" response = nothing # placeholder for show when error msg show up - #[PENDING] add 1) 3 decisions samples 2) compare and choose the best decision (correct tolls etc) - llmkwargs=Dict( - :num_ctx => 32768, - :temperature => 0.5, - ) - for attempt in 1:maxattempt if attempt > 1 println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - QandA = generatequestion(a, a.func[:text2textInstructLLM], timeline) - systemmsg = + # QandA = generatequestion(a, a.func[:text2textInstructLLM], timeline) + + context = """ - Your name is $(a.name). You are a helpful English-speaking assistant, acting as a polite, website-based sommelier for $(a.retailername)'s wine store. - Your goal includes: - 1) Establish a connection with the customer by greeting them warmly - 2) Guide them to select the best wines only from your store's inventory that align with their preferences - - Your responsibility includes: - 1) Make an informed decision about what you need to do to achieve the goal - 2) Thanks the user when they don't need any further assistance and invite them to comeback next time - - Your responsibility does NOT includes: - 1) Requesting the user to place an order, make a purchase, or confirm the order. These are the job of our sales team at the store. - 2) Processing sales orders or engaging in any other sales-related activities. These are the job of our sales team at the store. - 3) Answering questions or offering additional services beyond those related to your store's wine recommendations such as discounts, quantity, rewards programs, promotions, delivery options, shipping, boxes, gift wrapping, packaging, personalized messages or something similar. These are the job of our sales team at the store. - - At each round of conversation, you will be given the following information: - Q&A: the question and answer you have asked yourself about the current situation - - You must follow the following guidelines: - - Focus on the latest event - - Generally speaking, your inventory has some wines from France, the United States, Australia, Spain, and Italy, but you won't know exactly until you check your inventory. - - All wines in your inventory are always in stock - - Approach each customer with open-ended questions to understand their preferences, budget, and occasion. This will help you guide the conversation naturally while gathering essential insights. Once you have this information, you can efficiently check your inventory for the best match. - - Do not ask the user about wine's flavor e.g. floral, citrusy, nutty or some thing similar as these terms cannot be used to search the database. - - Once the user has selected their wine, ask the user if they need any further assistance. Do not offer any additional services. If the user doesn't need any further assistance, say goodbye and invite them to come back next time. - - Spicy foods should not be paired with medium and full-bodied red wines. - - You should follow the following guidelines: - - When searching an inventory, search as broadly as possible based on the information you have gathered so far. - - Encourage the customer to explore different options and try new things. - - If you are unable to locate the desired item after multiple attempts, it may not be available in your inventory. In such cases, inform the user that the item is unavailable and suggest an alternative instead. - - For your information: - - Your store carries only wine. - - Vintage 0 means non-vintage. - - All wine in your inventory has no organic, sustainability and sulfite information. - - You should then respond to the user with interleaving Thought, Plan, Action_name, Action_input: - 2) Plan: Based on the current situation, state a complete action plan to complete the task. Be specific. - 3) Action_name: (Typically corresponds to the execution of the first step in your plan) Can be one of the following tool names: - - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific. - - CHECKINVENTORY allows you to check information about wines you want in your inventory's database. The input is search criteria. Supported search parameters include: wine price, winery, name, vintage, region, country, type, grape varietal, tasting notes, occasion, food pairing, intensity, tannin, sweetness, and acidity. - Example query: "Dry, full-bodied red wine from Burgundy, France or Tuscany, Italy. Merlot varietal. price 100 to 1000 USD." - - PRESENTBOX which you can use to present wines you have found in your inventory to the user. The input are wine names that you want to present. - - ENDCONVERSATION which you can use to properly end the conversation with the user. Input is "NA". - 4) Action_input: The input to the action you are about to perform. This should be aligned with the plan - - You should only respond in format as described below: - Plan: ... - Action_name: ... - Action_input: ... - - Let's begin! - """ - - assistantinfo = - """ - - Q&A: $QandA - P.S. $errornote - + + + - CHATBOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific. + - CHECKINVENTORY allows you to check information about wines you want in your inventory's database. The input must be supported search criteria includeing: wine price, winery, name, vintage, region, country, type, grape varietal, tasting notes, occasion, food pairing, intensity, tannin, sweetness, and acidity. + Example query: "Dry, full-bodied red wine from Burgundy, France or Tuscany, Italy. Merlot varietal. price 100 to 1000 USD." + - PRESENTBOX which you can use to present wines you have found in your inventory to the user. The input are wine names that you want to present. + - ENDCONVERSATION which you can use to properly end the conversation with the user. Input is "NA". + + Remark: $errornote + Database search result: $database_search_result + """ unformatPrompt = @@ -262,47 +262,73 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 Dict(:name => "system", :text => systemmsg), ] - unformatPrompt = vcat(unformatPrompt, recentEventsDict) + unformatPrompt = vcat(unformatPrompt, recentchat) # put in model format prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) # add info - prompt = prompt * assistantinfo + prompt = prompt * context - response = a.func[:text2textInstructLLM](prompt; senderId=a.id, llmkwargs=llmkwargs) + response = a.func[:text2textInstructLLM](prompt; senderId=a.id) + response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) response = GeneralUtils.remove_french_accents(response) + # response = replace(response, '$'=>"USD") think, response = GeneralUtils.extractthink(response) - response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName; ) - - # check if response contain more than one functions from ["CHATBOX", "CHECKINVENTORY", "ENDCONVERSATION"] - count = 0 - for i ∈ ["CHATBOX", "CHECKINVENTORY", "PRESENTBOX", "ENDCONVERSATION"] - if occursin(i, response) - count += 1 - end - end - if count > 1 - errornote = "You must use only one function" - println("\nERROR YiemAgent decisionMaker() $errornote\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + + responsedict = nothing + try + responsedict = copy(JSON3.read(response)) + catch + println("\nERROR YiemAgent decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - # check whether response has all header - detected_kw = GeneralUtils.detect_keyword(header, response) - kwvalue = [i for i in values(detected_kw)] - zeroind = findall(x -> x == 0, kwvalue) - missingkeys = [header[i] for i in zeroind] - if 0 ∈ values(detected_kw) + # check whether all answer's key points are in responsedict + _responsedictKey = keys(responsedict) + responsedictKey = [i for i in _responsedictKey] # convert into a list + is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] + + if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) + errornote = "Your previous attempt has more key points than answer's required key points." + println("\nERROR YiemAgent decisionMaker() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + elseif !all(is_requiredKeys_in_responsedictKey) + zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) + missingkeys = [requiredKeys[i] for i in zeroind] errornote = "$missingkeys are missing from your previous response" - println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "Your previous attempt has duplicated points" - println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR YiemAgent decisionMaker() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) + # # check if response contain more than one functions from ["CHATBOX", "CHECKINVENTORY", "ENDCONVERSATION"] + # count = 0 + # for i ∈ ["CHATBOX", "CHECKINVENTORY", "PRESENTBOX", "ENDCONVERSATION"] + # if occursin(i, response) + # count += 1 + # end + # end + # if count > 1 + # errornote = "You must use only one function" + # println("\nERROR YiemAgent decisionMaker() $errornote\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end + + # # check whether response has all header + # detected_kw = GeneralUtils.detect_keyword(header, response) + # kwvalue = [i for i in values(detected_kw)] + # zeroind = findall(x -> x == 0, kwvalue) + # missingkeys = [header[i] for i in zeroind] + # if 0 ∈ values(detected_kw) + # errornote = "$missingkeys are missing from your previous response" + # println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # elseif sum(values(detected_kw)) > length(header) + # errornote = "Your previous attempt has duplicated points" + # println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end + + # responsedict = GeneralUtils.textToDict(response, header; + # dictKey=dictkey, symbolkey=true) if responsedict[:action_name] ∉ ["CHATBOX", "CHECKINVENTORY", "PRESENTBOX", "ENDCONVERSATION"] errornote = "Your previous attempt didn't use the given functions" @@ -310,29 +336,29 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 continue end - checkFlag = false - for i ∈ Symbol.(dictkey) - if length(responsedict[i]) == 0 - errornote = "$i is empty" - println("\nERROR YiemAgent decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - checkFlag = true - break - end - end - checkFlag == true ? continue : nothing + # checkFlag = false + # for i ∈ Symbol.(dictkey) + # if length(responsedict[i]) == 0 + # errornote = "$i is empty" + # println("\nERROR YiemAgent decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # checkFlag = true + # break + # end + # end + # checkFlag == true ? continue : nothing - # check if there are more than 1 key per categories - checkFlag = false - for i ∈ Symbol.(dictkey) - matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i) - if length(matchkeys) > 1 - errornote = "Your previous attempt has more than one key per categories" - println("\nERROR YiemAgent decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - checkFlag = true - break - end - end - checkFlag == true ? continue : nothing + # # check if there are more than 1 key per categories + # checkFlag = false + # for i ∈ Symbol.(dictkey) + # matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i) + # if length(matchkeys) > 1 + # errornote = "Your previous attempt has more than one key per categories" + # println("\nERROR YiemAgent decisionMaker() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # checkFlag = true + # break + # end + # end + # checkFlag == true ? continue : nothing # # check if action_name = CHECKINVENTORY and action_input has the words "pairs well" or # # "pair well" in it because it is not a valid query. @@ -375,7 +401,7 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 delete!(responsedict, :mentioned_winery) responsedict[:systemmsg] = systemmsg responsedict[:unformatPrompt] = unformatPrompt - responsedict[:QandA] = QandA + # responsedict[:QandA] = QandA # check whether responsedict[:action_input] is the same as previous dialogue if responsedict[:action_input] == a.chathistory[end][:text] @@ -384,8 +410,47 @@ function decisionMaker(a::T; recent::Integer=10, maxattempt=10 continue end - # println("\n$prompt", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # println("\n$response") + println("\nYiemAgent decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\n$response") + + #[WORKING] + evaluationdict = evaluator(a, timeline, responsedict, context) + if evaluationdict[:good_decision] == "no" + mentor_comment = evaluationdict[:suggestion] + errornote = "Your previous attempt was not good enough. Please try again. Here is the mentor's suggestion: $mentor_comment" + println("\nERROR YiemAgent decisionMaker() - $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + end + + # store for later training + responsedict[:system] = systemmsg + responsedict[:recentchat] = recentchat + responsedict[:prompt] = prompt + responsedict[:context] = context + responsedict[:think] = think + # responsedict[:QandA] = QandA + + # # save to filename ./log/decisionlog.txt + # println("\nsaving YiemAgent decisionMaker() to disk ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # filename = "agent_decision_log_$(a.id).json" + # filepath = "/appfolder/app/log/$filename" + # # check whether there is a file path exists before writing to it + # if !isfile(filepath) + # decisionlist = [responsedict] + # println("Creating file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # else + # # read the file and append new data + # decisionlist = copy(JSON3.read(filepath)) + # push!(decisionlist, responsedict) + # println("Appending new data to file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # end + # println("\nYiemAgent decisionMaker() saved to disk is done. agent $(a.id)") responsedict[:prompt] = prompt return responsedict @@ -735,153 +800,169 @@ julia> # Signature """ -function evaluator(state::T1, text2textInstructLLM::Function - ) where {T1<:AbstractDict} +function evaluator(a::T1, timeline, decisiondict, evaluateecontext + ) where {T1<:agent} systemmsg = """ - You are a helpful assistant that analyzes agent's trajectory to find solutions and observations (i.e., the results of actions) to answer the user's questions. - - Definitions: - "question" is the user's question - "understanding" is agent's understanding about the current situation - "reasoning" is agent's step-by-step reasoning about the current situation - "plan" is agent's plan to complete the task from the current situation - "action_name" is the name of the action taken, which can be one of the following functions: - - RUNSQL, which you can use to execute SQL against the database. Action_input for this function must be a single SQL query to be executed against the database. - For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator. - Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'. - "action_input" is the input to the action - "observation" is result of the preceding immediate action - - - Trajectory: ... - Error_note: error note from your previous attempt - - - - - When the search returns no result, validate whether the SQL query makes sense before accepting it as a valid answer. - - + + - You are a master sommelier of an online wine store. + + + - Under your supervision, a trainee sommelier is engaging with a store customer. Each time the customer speaks, the trainee will assess the situation, determine the next course of action, and pause to await your guidance before proceeding. + + + - To improve a trainee sommelier decision based on the store policy and guidelines. + + + - trajectory: A conversation between your trainee and the customer that have occurred up until now + - evaluatee_context: The context that evaluatee use to make a decision + - evaluatee_decision: The decision made by the evaluatee, consists of the following elements: + "plan" is the trainee's plan + "action_name" is the name of the action taken, which can be one of the available tool name. + "action_input" is the input to the action. + + + - Generally speaking, the store inventory has some wines from France, the United States, Australia, Spain, and Italy, but you won't know exactly until you check your inventory. + - If you found wines in the store's database, they are in stock + - Approach each customer with open-ended questions to understand their preferences, budget, and occasion. Once you have these information, you can check your inventory. + - Do not ask the user about wine's flavor e.g. floral, citrusy, nutty or some thing similar as these terms cannot be used to search the database. Other wine characteristics useful for CHECKINVENTORY tools are allowed. + - Once the user has selected their wine, ask the user if they need any further assistance. Do not offer any additional services. If the user doesn't need any further assistance, say goodbye and invite them to come back next time. + - Spicy foods should not be paired with medium and full-bodied red wines. + - We do not sell organic, sustainability and sulfite wine. + + + - Encourage the customer to explore different options and try new things. + - If you are unable to locate the desired item in the database after multiple attempts, it may not be available in your inventory. In such cases, inform the user that the item is unavailable and suggest an alternative instead. + - Your store carries only wine. + - Vintage 0 means non-vintage. + + + - Use only infomation provided by the store policy and guidelines as a bedrocks for your response. + + + - N/A + - 1) Trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. - - Evaluate the correctness of each section and the overall trajectory based on the given question. - - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation. - - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. - - Do not generate additional thoughts or actions. - 2) Answer_evaluation: - - Focus only on the matter mentioned in the question and comprehensively analyze how the latest observation's details addresses the question - 3) Accepted_as_answer: Decide whether the latest observation's content answers the question. Can be "Yes" or "No" - Bad example (The observation didn't answers the question): - question: Find cars with 4 wheels. - observation: There are an apple in the table. - Good example (The observation answers the question): - question: Find cars with a stereo. - observation: There are 1 cars in the table. 1) brand: Toyota, model: yaris, color: black. - 4) Score: Correctness score s where s is a single integer between 0 to 9. - For example: - - 0 indicates that both the trajectory is incorrect, failed or errors and the observation is incorrect or failed - - 4 indicates that the trajectory are correct but the observation is incorrect or failed - - 5 indicates that the trajectory are correct, but no results are returned. - - 6 indicates that the trajectory are correct, but the observation's content doesn't directly answer the question - - 8 indicates that both the trajectory are correct, and the observation's content directly answers the question. - - 9 indicates a perfect perfomance. Both the trajectory are correct, and the observation's content directly answers the question, surpassing your expectations. - 5) Suggestion: if accepted_as_answer is "No", provide suggestion. - + 1) trajectory_evaluation: Analyze the trajectory of a solution to answer the user's original question. + - Evaluate the correctness of each section and the overall trajectory based on the given question. + - Provide detailed reasoning and analysis, focusing on the latest thought, action, and observation. + - Incomplete trajectory are acceptable if the thoughts and actions up to that point are correct, even if the final answer isn't reached. + - Do not generate additional thoughts or actions. + 2) decision_evaluation: + - Examine how the trainee's decisions align with the store's policies and guidelines before proceeding. + 3) good_decision: Decide whether the decision make sense under the circumstances. Can be "yes" or "no" + 4) suggestion: Based store policy and guidelines, provide a suggestion for the immediate decision step only. - - Trajectory_evaluation: ... - Answer_evaluation: ... - Accepted_as_answer: ... - Score: ... - Suggestion: ... + + + { + "trajectory_evaluation": "..." + "decision_evaluation": "..." + "good_decision": "..." + "suggestion": "..." + } Let's begin! """ - - thoughthistory = "" - for (k, v) in state[:thoughtHistory] - thoughthistory *= "$k: $v\n" - end - + requiredKeys = [:trajectory_evaluation, :decision_evaluation, :good_decision, :suggestion] errornote = "N/A" for attempt in 1:10 - errorFlag = false + evaluateecontext = replace(evaluateecontext, "" => "") + evaluateecontext = replace(evaluateecontext, "" => "") - usermsg = + context = """ - Trajectory: $thoughthistory - Error_note: $errornote + + + $timeline + + + $evaluateecontext + + + {plan: $(decisiondict[:plan]), action_name: $(decisiondict[:action_name]), action_input: $(decisiondict[:action_input])} + + P.S. $errornote + """ - _prompt = - [ - Dict(:name=> "system", :text=> systemmsg), - Dict(:name=> "user", :text=> usermsg) - ] + unformatPrompt = + [ + Dict(:name => "system", :text => systemmsg), + ] # put in model format - prompt = GeneralUtils.formatLLMtext(_prompt, "qwen3") + prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) + # add info + prompt = prompt * context + println("") + println(prompt) - header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"] - dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"] - - response = text2textInstructLLM(prompt; modelsize="medium", senderId=a.id) - - # sometime LLM output something like **Comprehension**: which is not expected - response = replace(response, "**"=>"") - response = replace(response, "***"=>"") - response = GeneralUtils.deFormatLLMtext(response, "qwen3") - - # check whether response has all header - detected_kw = GeneralUtils.detect_keyword(header, response) - if 0 ∈ values(detected_kw) - errornote = "\nSQL evaluator() response does not have all header" - println("\nERROR YiemAgent decisionMaker() $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "\nSQL evaluator() response has duplicated header" - println("\nERROR YiemAgent decisionMaker() $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) + response = a.func[:text2textInstructLLM](prompt; senderId=a.id) + response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) + response = GeneralUtils.remove_french_accents(response) + # response = replace(response, '$'=>"USD") + think, response = GeneralUtils.extractthink(response) - responsedict[:score] = responsedict[:score][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number. + responsedict = nothing try - responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5 + responsedict = copy(JSON3.read(response)) catch + println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - accepted_as_answer::AbstractString = responsedict[:accepted_as_answer] + # check whether all answer's key points are in responsedict + _responsedictKey = keys(responsedict) + responsedictKey = [i for i in _responsedictKey] # convert into a list + is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] - if accepted_as_answer ∉ ["Yes", "No"] # [PENDING] add errornote into the prompt - error("generated accepted_as_answer has wrong format") + if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) + errornote = "Your previous attempt has more key points than answer's required key points." + println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + elseif !all(is_requiredKeys_in_responsedictKey) + zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) + missingkeys = [requiredKeys[i] for i in zeroind] + errornote = "$missingkeys are missing from your previous response" + println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue end - # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension) - state[:evaluation] = "$(responsedict[:trajectory_evaluation]) $(responsedict[:answer_evaluation])" - state[:evaluationscore] = responsedict[:score] - state[:accepted_as_answer] = responsedict[:accepted_as_answer] - state[:suggestion] = responsedict[:suggestion] + # if accepted_as_answer ∉ ["yes", "no"] # [PENDING] add errornote into the prompt + # error("generated accepted_as_answer has wrong format") + # end - # mark as terminal state when the answer is achieved - if accepted_as_answer == "Yes" - - # mark the state as terminal state because the evaluation say so. - state[:isterminal] = true - - # evaluation score as reward because different answers hold different value for the user. - state[:reward] = responsedict[:score] - end - println("\nERROR Evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nEvaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") pprintln(Dict(responsedict)) - return responsedict[:score] + # # read sessionId + # sessionid = a.id + # # save to filename ./log/decisionlog.txt + # println("saving SQLLLM evaluator() to disk") + # filename = "agent_evaluator_log_$(sessionid[:id]).json" + # filepath = "/appfolder/app/log/$filename" + # # check whether there is a file path exists before writing to it + # if !isfile(filepath) + # decisionlist = [responsedict] + # println("Creating file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # else + # # read the file and append new data + # decisionlist = copy(JSON3.read(filepath)) + # push!(decisionlist, responsedict) + # println("Appending new data to file $filepath") + # open(filepath, "w") do io + # JSON3.pretty(io, decisionlist) + # end + # end + + return responsedict end error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>") end @@ -892,7 +973,7 @@ end # Arguments `a::agent` an agent - + # Return None @@ -1119,16 +1200,14 @@ function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} wh elseif actionname == "CHECKINVENTORY" if rawresponse !== nothing - vd = GeneralUtils.dfToVectorDict(rawresponse) + vd = GeneralUtils.dfToVectorDict(rawresponse) # comes in dataframe format a.memory[:shortmem][:found_wine] = vd # used by decisionMaker() as a short note - #[PENDING] a.memory[:shortmem][:available_wine] should be OrderedDict instead of vector if dict so that no duplicate item are listed? - if length(a.memory[:shortmem][:available_wine]) != 0 - a.memory[:shortmem][:available_wine] = vcat(a.memory[:shortmem][:available_wine], vd) + #[PENDING] a.memory[:shortmem][:available_wine] should be OrderedDict instead of vector dict so that no duplicate item are listed? + if length(a.memory[:shortmem][:db_search_result]) != 0 + a.memory[:shortmem][:db_search_result] = vcat(a.memory[:shortmem][:db_search_result], vd) else - a.memory[:shortmem][:available_wine] = vd + a.memory[:shortmem][:db_search_result] = vd end - else - println("checkinventory return nothing") end push!(a.memory[:events], @@ -1142,6 +1221,7 @@ function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} wh outcome= "This is what I found:, $result" ) ) + else error("condition is not defined ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end @@ -1150,7 +1230,7 @@ function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} wh end -function presentbox(a::sommelier, thoughtDict) +function presentbox(a::sommelier, thoughtDict; maxtattempt::Integer=10) systemmsg = """ Your profile: @@ -1160,102 +1240,123 @@ function presentbox(a::sommelier, thoughtDict) Your mission: Present the wines to the customer in a way that keep the conversation smooth and engaging. At each round of conversation, you will be given the following information: - Additional info: additional information + Database search result: the result of a database search using SQL commands you have found so far Chat history: your ongoing conversation with the user - Wine name: name if wines you found. + Wine name: name of wines you are going to introduce. You should follow the following guidelines: - Provide detailed introductions of the wines you've found to the user. - Explain how the wine could match the user's intention and what its effects might mean for the user's experience. - If multiple wines are available, highlight their differences and provide a comprehensive comparison of how each option aligns with the user's intention and what the potential effects of each option could mean for the user's experience. - Provide your personal recommendation and provide a brief explanation of why you recommend it. You should then respond to the user with: - Dialogue: Your presentation to the user + dialogue: Your presentation to the user You should only respond in format as described below: - Dialogue: ... + { + "dialogue": "..." + } Let's begin! """ - - header = ["Dialogue:"] - dictkey = ["dialogue"] - - # a.memory[:shortmem][:available_wine] is a vector of dictionary - context = - if length(a.memory[:shortmem][:available_wine]) != 0 - "Wines previously found in your inventory: $(availableWineToText(a.memory[:shortmem][:available_wine]))" - else - "N/A" - end + requiredKeys = [:dialogue] + database_search_result = + if length(a.memory[:shortmem][:db_search_result]) != 0 + availableWineToText(a.memory[:shortmem][:db_search_result]) + else + "N/A" + end chathistory = chatHistoryToText(a.chathistory) errornote = "N/A" response = nothing # placeholder for show when error msg show up + + # yourthought = "$(thoughtDict[:thought]) $(thoughtDict[:plan])" # yourthought1 = nothing - for attempt in 1:10 + for attempt in 1:maxtattempt - if attempt > 1 # use to prevent LLM generate the same respond over and over - println("\nYiemAgent presentbox() attempt $attempt/10 ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # yourthought1 = paraphrase(a.func[:text2textInstructLLM], yourthought) - else - # yourthought1 = yourthought - end - - usermsg = + context = """ - $errornote - Additional info: $context + + Database search result: $database_search_result Chat history: $chathistory Wine name: $(thoughtDict[:action_input]) + P.S. $errornote + """ - _prompt = + unformatPrompt = [ Dict(:name => "system", :text => systemmsg), - Dict(:name => "user", :text => usermsg) ] # put in model format - prompt = GeneralUtils.formatLLMtext(_prompt, a.llmFormatName) - # update this function to use new llm + prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) + # add info + prompt = prompt * context + response = a.func[:text2textInstructLLM](prompt; senderId=a.id) response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) - think, response = GeneralUtils.extractthink(response) - # check whether response has not-allowed words - notallowed = ["respond:", "user>", "user:"] - detected_kw = GeneralUtils.detect_keyword(notallowed, response) - # list all keys that have 1 value in detected_kw dictionary - k = [key for (key, value) in detected_kw if value == 1] - if length(k) > 0 - errornote = "In your previous attempt, you have $k in your response which is not allowed." - println("\nERROR YiemAgent presentbox() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - response = GeneralUtils.remove_french_accents(response) + # response = replace(response, '$'=>"USD") + think, response = GeneralUtils.extractthink(response) + response = replace(response, '*'=>"") response = replace(response, '$' => "USD") response = replace(response, '`' => "") response = replace(response, "<|eot_id|>"=>"") - response = GeneralUtils.remove_french_accents(response) - response = GeneralUtils.deFormatLLMtext(response, "qwen3") - - # check whether response has all header - detected_kw = GeneralUtils.detect_keyword(header, response) - if 0 ∈ values(detected_kw) - errornote = "$missingkeys are missing from your previous response" - println("\nERROR YiemAgent presentbox() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "\nYour previous attempt has duplicated points according to the required response format" - println("\nERROR YiemAgent presentbox() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + + responsedict = nothing + try + responsedict = copy(JSON3.read(response)) + catch + println("\nERROR YiemAgent presentbox() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) + # check whether all answer's key points are in responsedict + _responsedictKey = keys(responsedict) + responsedictKey = [i for i in _responsedictKey] # convert into a list + is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] + + if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) + errornote = "Your previous attempt has more key points than answer's required key points." + println("\nERROR YiemAgent presentbox() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + elseif !all(is_requiredKeys_in_responsedictKey) + zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) + missingkeys = [requiredKeys[i] for i in zeroind] + errornote = "$missingkeys are missing from your previous response" + println("\nERROR YiemAgent presentbox() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + end + + # # check whether response has not-allowed words + # notallowed = ["respond:", "user>", "user:"] + # detected_kw = GeneralUtils.detect_keyword(notallowed, response) + # # list all keys that have 1 value in detected_kw dictionary + # k = [key for (key, value) in detected_kw if value == 1] + # if length(k) > 0 + # errornote = "In your previous attempt, you have $k in your response which is not allowed." + # println("\nERROR YiemAgent presentbox() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end + + # # check whether response has all header + # detected_kw = GeneralUtils.detect_keyword(header, response) + # if 0 ∈ values(detected_kw) + # errornote = "$missingkeys are missing from your previous response" + # println("\nERROR YiemAgent presentbox() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # elseif sum(values(detected_kw)) > length(header) + # errornote = "\nYour previous attempt has duplicated points according to the required response format" + # println("\nERROR YiemAgent presentbox() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end + + # responsedict = GeneralUtils.textToDict(response, header; + # dictKey=dictkey, symbolkey=true) # check if Context: is in dialogue if occursin("Context:", responsedict[:dialogue]) @@ -1321,7 +1422,7 @@ julia> # Signature """ -function generatechat(a::sommelier, thoughtDict) +function generatechat(a::sommelier, thoughtDict; maxattempt::Integer=10) systemmsg = """ Your role: @@ -1335,7 +1436,6 @@ function generatechat(a::sommelier, thoughtDict) - Processing sales orders or engaging in any other sales-related activities. These are the job of our sales team at the store. - Answering questions or offering additional services beyond those related to your store's wine recommendations such as discounts, quantity, rewards programs, promotions, delivery options, shipping, boxes, gift wrapping, packaging, personalized messages or something similar. These are the job of our sales team at the store. At each round of conversation, you will be given the following: - Additional info: ... Your ongoing conversation with the user: ... Your thoughts: Your current thoughts in your mind You must follow the following guidelines: @@ -1345,68 +1445,92 @@ function generatechat(a::sommelier, thoughtDict) - If the user interrupts, prioritize the user - Be honest You should then respond to the user with: - Dialogue: what you want to say to the user - You should only respond in format as described below: - Dialogue: ... + dialogue: what you want to say to the user + You should only respond in JSON format as described below: + { + "dialogue": "..." + } Here are some examples: - Additional info: "Car previously found in your inventory: 1) Toyota Camry 2020 2) Honda Civic 2021 3) Ford Mustang 2022" - Your thoughts: "I should recommend the car we have found in our inventory to the user." Your ongoing conversation with the user: "user> hello, I need a new car\n" - Dialogue: "We have a variety of cars available, including the Toyota Camry 2020, the Honda Civic 2021, and the Ford Mustang 2022. Which one would you like to see?" + Your thoughts: "I should recommend the car we have found in our inventory to the user." + {"dialogue": "We have a variety of cars available, including the Toyota Camry 2020, the Honda Civic 2021, and the Ford Mustang 2022. Which one would you like to see?"} Let's begin! """ - - header = ["Dialogue:"] - dictkey = ["dialogue"] + requiredKeys = [:dialogue] # a.memory[:shortmem][:available_wine] is a vector of dictionary - context = - if length(a.memory[:shortmem][:available_wine]) != 0 - "Wines previously found in your inventory: $(availableWineToText(a.memory[:shortmem][:available_wine]))" - else - "N/A" - end + # context = + # if length(a.memory[:shortmem][:available_wine]) != 0 + # "Wines previously found in your inventory: $(availableWineToText(a.memory[:shortmem][:available_wine]))" + # else + # "N/A" + # end chathistory = chatHistoryToText(a.chathistory) errornote = "N/A" response = nothing # placeholder for show when error msg show up yourthought = thoughtDict[:plan] - yourthought1 = nothing + # yourthought1 = nothing - llmkwargs=Dict( - :num_ctx => 32768, - :temperature => 0.5, - ) + for attempt in 1:maxattempt + # if attempt > 1 # use to prevent LLM generate the same respond over and over + # println("\nYiemAgent generatchat() attempt $attempt/10 ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # yourthought1 = paraphrase(a.func[:text2textInstructLLM], yourthought) + # else + # yourthought1 = yourthought + # end - for attempt in 1:10 - if attempt > 1 # use to prevent LLM generate the same respond over and over - println("\nYiemAgent generatchat() attempt $attempt/10 ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - yourthought1 = paraphrase(a.func[:text2textInstructLLM], yourthought) - else - yourthought1 = yourthought - end - - usermsg = + context = """ - Additional info: $context + Your ongoing conversation with the user: $chathistory - Your thoughts: $yourthought1 + Your thoughts: $yourthought P.S. $errornote + """ - _prompt = + unformatPrompt = [ Dict(:name => "system", :text => systemmsg), - Dict(:name => "user", :text => usermsg) ] # put in model format - prompt = GeneralUtils.formatLLMtext(_prompt, a.llmFormatName) - response = a.func[:text2textInstructLLM](prompt; llmkwargs=llmkwargs) + prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) + # add info + prompt = prompt * context + + response = a.func[:text2textInstructLLM](prompt; senderId=a.id) response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) + response = GeneralUtils.remove_french_accents(response) + # response = replace(response, '$'=>"USD") think, response = GeneralUtils.extractthink(response) + + responsedict = nothing + try + responsedict = copy(JSON3.read(response)) + catch + println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + end + + # check whether all answer's key points are in responsedict + _responsedictKey = keys(responsedict) + responsedictKey = [i for i in _responsedictKey] # convert into a list + is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] + + if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) + errornote = "Your previous attempt has more key points than answer's required key points." + println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + elseif !all(is_requiredKeys_in_responsedictKey) + zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) + missingkeys = [requiredKeys[i] for i in zeroind] + errornote = "$missingkeys are missing from your previous response" + println("\nERROR YiemAgent generatechat() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + end # sometime the model response like this "here's how I would respond: ..." if occursin("respond:", response) @@ -1425,30 +1549,32 @@ function generatechat(a::sommelier, thoughtDict) response = replace(response, "<|eot_id|>"=>"") response = GeneralUtils.remove_french_accents(response) - # check whether response has all header - detected_kw = GeneralUtils.detect_keyword(header, response) - kwvalue = [i for i in values(detected_kw)] - zeroind = findall(x -> x == 0, kwvalue) - missingkeys = [header[i] for i in zeroind] - if 0 ∈ values(detected_kw) - errornote = "$missingkeys are missing from your previous response" - println("\nERROR YiemAgent generatechat() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "\nYour previous attempt has duplicated points according to the required response format" - println("\nERROR YiemAgent generatechat() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end + - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) + # # check whether response has all header + # detected_kw = GeneralUtils.detect_keyword(header, response) + # kwvalue = [i for i in values(detected_kw)] + # zeroind = findall(x -> x == 0, kwvalue) + # missingkeys = [header[i] for i in zeroind] + # if 0 ∈ values(detected_kw) + # errornote = "$missingkeys are missing from your previous response" + # println("\nERROR YiemAgent generatechat() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # elseif sum(values(detected_kw)) > length(header) + # errornote = "\nYour previous attempt has duplicated points according to the required response format" + # println("\nERROR YiemAgent generatechat() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end - # check if Context: is in dialogue - if occursin("Context:", responsedict[:dialogue]) - errornote = "Your previous response contains 'Context:' which is not allowed" - println("\nERROR YiemAgent generatechat() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end + # responsedict = GeneralUtils.textToDict(response, header; + # dictKey=dictkey, symbolkey=true) + + # # check if Context: is in dialogue + # if occursin("Context:", responsedict[:dialogue]) + # errornote = "Your previous response contains 'Context:' which is not allowed" + # println("\nERROR YiemAgent generatechat() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end # println("\nYiemAgent generatechat() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # pprintln(Dict(responsedict)) @@ -1546,7 +1672,7 @@ function generatechat(a::virtualcustomer; ) recent_ind = GeneralUtils.recentElementsIndex(length(a.memory[:events]), recentEventNum; includelatest=true) recentevents = a.memory[:events][recent_ind] - recentEventsDict = createEventsLog(recentevents; eventindex=recent_ind) + recentEventsDict = createEventsLog(recentevents; index=recent_ind) response = nothing # placeholder for show when error msg show up errornote = "N/A" header = ["Dialogue:", "Role"] @@ -1560,11 +1686,11 @@ function generatechat(a::virtualcustomer; if attempt > 1 println("\nYiemAgent generatechat() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - assistantinfo = + context = """ - + P.S. $errornote - + """ unformatPrompt = @@ -1575,7 +1701,7 @@ function generatechat(a::virtualcustomer; # put in model format prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) # add info - prompt = prompt * assistantinfo + prompt = prompt * context response = a.func[:text2textInstructLLM](prompt; llmkwargs=llmkwargs, senderId=a.id) response = replace(response, "<|im_start|>"=> "") @@ -1618,14 +1744,13 @@ function generatechat(a::virtualcustomer; # println("\n$prompt", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println("\n $response") - return responsedict[:dialogue] + return responsedict[:dialogue] end error("generatechat failed to generate a response") end function generatequestion(a, text2textInstructLLM::Function, timeline)::String - systemmsg = """ Your role: @@ -1733,12 +1858,13 @@ function generatequestion(a, text2textInstructLLM::Function, timeline)::String header = ["Q1:"] dictkey = ["q1"] - context = - if length(a.memory[:shortmem][:available_wine]) != 0 - "Available wines you've found in your inventory so far: $(availableWineToText(a.memory[:shortmem][:available_wine]))" - else - "N/A" - end + # context = + # if length(a.memory[:shortmem][:available_wine]) != 0 + # "Available wines you've found in your inventory so far: $(availableWineToText(a.memory[:shortmem][:available_wine]))" + # else + # "N/A" + # end + database_search_result = a.memory[:shortmem][:db_search_result] # recent_ind = GeneralUtils.recentElementsIndex(length(a.memory[:events]), recent) # recentevents = a.memory[:events][recent_ind] @@ -1775,7 +1901,7 @@ function generatequestion(a, text2textInstructLLM::Function, timeline)::String usermsg = """ - Additional info: $context + Additional info: $database_search_result Your recent events: $timeline P.S. $errornote """ diff --git a/src/llmfunction.jl b/src/llmfunction.jl index d58e08c..78a522f 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -339,7 +339,7 @@ function extractWineAttributes_1(a::T1, input::T2; maxattempt=10 As a helpful sommelier, your task is to extract the user information from the user's query as much as possible to fill out user's preference form. At each round of conversation, the user will give you the following: - User's query: ... + - The query: the query provided by the user. You must follow the following guidelines: - If specific information required in the preference form is not available in the query or there isn't any, mark with "N/A" to indicate this. @@ -347,121 +347,167 @@ function extractWineAttributes_1(a::T1, input::T2; maxattempt=10 - Do not generate other comments. You should then respond to the user with: - Wine_name: name of the wine - Winery: name of the winery - Vintage: the year of the wine - Region: a region (NOT a country) where the wine is produced, such as Burgundy, Napa Valley, etc - Country: a country where wine is produced. Can be "Austria", "Australia", "France", "Germany", "Italy", "Portugal", "Spain", "United States" - Wine_type: can be one of: "red", "white", "sparkling", "rose", "dessert" or "fortified" - Grape_varietal: the name of the primary grape used to make the wine - Tasting_notes: a word describe the wine's flavor, such as "butter", "oak", "fruity", "raspberry", "earthy", "floral", etc - Wine_price_range: price range of wine. Example: For price 10-20, price range will be "10 to 20". For price 100, price range will be 0 to 100. - Occasion: the occasion the user is having the wine for - Food_to_be_paired_with_wine: food that the user will be served with the wine such as poultry, fish, steak, etc + wine_name: name of the wine + winery: name of the winery + vintage: the year of the wine + region: a region (NOT a country) where the wine is produced, such as Burgundy, Napa Valley, etc + country: a country where wine is produced. Can be "Austria", "Australia", "France", "Germany", "Italy", "Portugal", "Spain", "United States" + wine_type: can be one of: "red", "white", "sparkling", "rose", "dessert" or "fortified" + grape_varietal: the name of the primary grape used to make the wine + tasting_notes: a word describe the wine's flavor, such as "butter", "oak", "fruity", "raspberry", "earthy", "floral", etc + wine_price_min: minimum price range of wine. Example: For wine price 20, wine_price_min will be 0. For wine price 10 to 100, wine_price_min will be 10. + wine_price_max: maximum price range of wine. Example: For wine price 20, wine_price_max will be 20. For wine price 10 to 100, wine_price_max will be 100. + occasion: the occasion the user is having the wine for + food_to_be_paired_with_wine: food that the user will be served with the wine such as poultry, fish, steak, etc - You should only respond in format as described below: - Wine_name: ... - Winery: ... - Vintage: ... - Region: ... - Country: ... - Wine_type: - Grape_varietal: ... - Tasting_notes: ... - Wine_price_range: ... - Occasion: ... - Food_to_be_paired_with_wine: ... + You should only respond in JSON format as described below: + { + "wine_name": "...", + "winery": "...", + "vintage": "...", + "region": "...", + "country": "...", + "wine_type": "...", + "grape_varietal": "...", + "tasting_notes": "...", + "wine_price_min": "...", + "wine_price_max": "...", + "occasion": "...", + "food_to_be_paired_with_wine": "..." + } Here are some example: User's query: red, Chenin Blanc, Riesling, 20 USD from Tuscany, Italy or Napa Valley, USA - Wine_name: N/A. Winery: N/A. Vintage: N/A. Region: Tuscany, Napa Valley. Country: Italy, United States. Wine_type: red, white. Grape_varietal: Chenin Blanc, Riesling. Tasting_notes: citrus. Wine_price_range: 0 to 20. Occasion: N/A. Food_to_be_paired_with_wine: N/A + { + "wine_name": "N/A", + "winery": "N/A", + "vintage": "N/A", + "region": "Tuscany, Napa Valley", + "country": "Italy, United States", + "wine_type": "red, white", + "grape_varietal": "Chenin Blanc, Riesling", + "tasting_notes": "citrus", + "wine_price_min": "0", + "wine_price_max": "20", + "occasion": "N/A", + "food_to_be_paired_with_wine": "N/A" + } User's query: Domaine du Collier Saumur Blanc 2019, France, white, Merlot - Winery: Domaine du Collier. Wine_name: Saumur Blanc. Vintage: 2019. Region: Saumur. Country: France. Wine_type: white. Grape_varietal: Merlot. Tasting_notes: plum. Wine_price_range: N/A. Occasion: N/A. Food_to_be_paired_with_wine: N/A. + { + "wine_name": "Saumur Blanc", + "winery": "Domaine du Collier", + "vintage": "2019", + "region": "Saumur", + "country": "France", + "wine_type": "white", + "grape_varietal": "Merlot", + "tasting_notes": "plum", + "wine_price_min": "N/A", + "wine_price_max": "N/A", + "occasion": "N/A", + "food_to_be_paired_with_wine": "N/A" + } Let's begin! """ - header = ["Wine_name:", "Winery:", "Vintage:", "Region:", "Country:", "Wine_type:", "Grape_varietal:", "Tasting_notes:", "Wine_price_range:", "Occasion:", "Food_to_be_paired_with_wine:"] - dictkey = ["wine_name", "winery", "vintage", "region", "country", "wine_type", "grape_varietal", "tasting_notes", "wine_price_range", "occasion", "food_to_be_paired_with_wine"] + requiredKeys = [:wine_name, :winery, :vintage, :region, :country, :wine_type, :grape_varietal, :tasting_notes, :wine_price_min, :wine_price_max, :occasion, :food_to_be_paired_with_wine] errornote = "N/A" - - llmkwargs=Dict( - :num_ctx => 32768, - :temperature => 0.2, - ) for attempt in 1:maxattempt usermsg = """ $input """ - assistantinfo = + context = """ - + P.S. $errornote - + + /no_think """ - _prompt = + unformatPrompt = [ Dict(:name=> "system", :text=> systemmsg), Dict(:name=> "user", :text=> usermsg) ] # put in model format - prompt = GeneralUtils.formatLLMtext(_prompt, a.llmFormatName) + prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) # add info - prompt = prompt * assistantinfo + prompt = prompt * context - response = a.func[:text2textInstructLLM](prompt; - modelsize="medium", llmkwargs=llmkwargs, senderId=a.id) - response = GeneralUtils.remove_french_accents(response) + response = a.func[:text2textInstructLLM](prompt; modelsize="medium", senderId=a.id) response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) + response = GeneralUtils.remove_french_accents(response) think, response = GeneralUtils.extractthink(response) - # check whether response has all header - detected_kw = GeneralUtils.detect_keyword(header, response) - kwvalue = [i for i in values(detected_kw)] - zeroind = findall(x -> x == 0, kwvalue) - missingkeys = [header[i] for i in zeroind] - if 0 ∈ values(detected_kw) + responsedict = nothing + try + responsedict = copy(JSON3.read(response)) + catch + println("\nERROR YiemAgent extractWineAttributes_1() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + end + + # check whether all answer's key points are in responsedict + _responsedictKey = keys(responsedict) + responsedictKey = [i for i in _responsedictKey] # convert into a list + is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] + + if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) + errornote = "Your previous attempt has more key points than answer's required key points." + println("\nERROR YiemAgent extractWineAttributes_1() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + continue + elseif !all(is_requiredKeys_in_responsedictKey) + zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) + missingkeys = [requiredKeys[i] for i in zeroind] errornote = "$missingkeys are missing from your previous response" - println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "Your previous attempt has duplicated points" - println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + println("\nERROR YiemAgent extractWineAttributes_1() $errornote --> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue end + + # # check whether response has all header + # detected_kw = GeneralUtils.detect_keyword(header, response) + # kwvalue = [i for i in values(detected_kw)] + # zeroind = findall(x -> x == 0, kwvalue) + # missingkeys = [header[i] for i in zeroind] + # if 0 ∈ values(detected_kw) + # errornote = "$missingkeys are missing from your previous response" + # println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # elseif sum(values(detected_kw)) > length(header) + # errornote = "Your previous attempt has duplicated points" + # println("\nERROR YiemAgent decisionMaker() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end - # check whether response has all answer's key points - detected_kw = GeneralUtils.detect_keyword(header, response) - if 0 ∈ values(detected_kw) - errornote = "In your previous attempts, the response does not have all answer's key points" - println("\nYiemAgent extractWineAttributes_1() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum(values(detected_kw)) > length(header) - errornote = "In your previous attempts, the response has duplicated answer's key points" - println("\nYiemAgent extractWineAttributes_1() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - println(response) - continue - end - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) + # # check whether response has all answer's key points + # detected_kw = GeneralUtils.detect_keyword(header, response) + # if 0 ∈ values(detected_kw) + # errornote = "In your previous attempts, the response does not have all answer's key points" + # println("\nYiemAgent extractWineAttributes_1() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # elseif sum(values(detected_kw)) > length(header) + # errornote = "In your previous attempts, the response has duplicated answer's key points" + # println("\nYiemAgent extractWineAttributes_1() Attempt $attempt $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # println(response) + # continue + # end + # responsedict = GeneralUtils.textToDict(response, header; + # dictKey=dictkey, symbolkey=true) delete!(responsedict, :thought) delete!(responsedict, :tasting_notes) delete!(responsedict, :occasion) delete!(responsedict, :food_to_be_paired_with_wine) - println(@__FILE__, " ", @__LINE__) - pprintln(responsedict) - # check if winery, wine_name, region, country, wine_type, grape_varietal's value are in the query because sometime AI halucinates checkFlag = false - for i in dictkey + for i in requiredKeys j = Symbol(i) if j ∉ [:thought, :tasting_notes, :occasion, :food_to_be_paired_with_wine] # in case j is wine_price it needs to be checked differently because its value is ranged @@ -759,7 +805,8 @@ function paraphrase(text2textInstructLLM::Function, text::String) Let's begin! """ - + #[WORKING] use JSON3 the same as extractWineAttributes_1 is better + #[WORKING] change this function to use the same format use decisionMater header = ["Paraphrase:"] dictkey = ["paraphrase"] diff --git a/src/type.jl b/src/type.jl index 871cec5..530238d 100644 --- a/src/type.jl +++ b/src/type.jl @@ -193,8 +193,7 @@ function sommelier( """ memory = Dict{Symbol, Any}( :shortmem=> OrderedDict{Symbol, Any}( - :available_wine=> [], - :found_wine=> [], # used by decisionMaker(). This is to prevent decisionMaker() keep presenting the same wines + :db_search_result=> Any[], ), :events=> Vector{Dict{Symbol, Any}}(), :state=> Dict{Symbol, Any}( diff --git a/src/util.jl b/src/util.jl index 488ec4a..2f18772 100644 --- a/src/util.jl +++ b/src/util.jl @@ -1,7 +1,7 @@ module util export clearhistory, addNewMessage, chatHistoryToText, eventdict, noises, createTimeline, - availableWineToText, createEventsLog + availableWineToText, createEventsLog, createChatLog using UUIDs, Dates, DataStructures, HTTP, JSON3 using GeneralUtils @@ -301,10 +301,10 @@ function createTimeline(events::T1; eventindex::Union{UnitRange, Nothing}=nothin for (i, event) in zip(ind, events) # If no outcome exists, format without outcome if event[:outcome] === nothing - timeline *= "Event_$i $(event[:subject])> $(event[:actioninput])\n" + timeline *= "Event_$i $(event[:subject])> action_name: $(event[:actionname]), action_input: $(event[:actioninput]), observation: Not done yet.\n" # If outcome exists, include it in formatting else - timeline *= "Event_$i $(event[:subject])> $(event[:actioninput]) $(event[:outcome])\n" + timeline *= "Event_$i $(event[:subject])> action_name: $(event[:actionname]), action_input: $(event[:actioninput]), observation: $(event[:outcome])\n" end end @@ -313,15 +313,15 @@ function createTimeline(events::T1; eventindex::Union{UnitRange, Nothing}=nothin end -function createEventsLog(events::T1; eventindex::Union{UnitRange, Nothing}=nothing +function createEventsLog(events::T1; index::Union{UnitRange, Nothing}=nothing ) where {T1<:AbstractVector} # Initialize empty log array log = Dict{Symbol, String}[] # Determine which indices to use - either provided range or full length ind = - if eventindex !== nothing - [eventindex...] + if index !== nothing + [index...] else 1:length(events) end @@ -338,7 +338,7 @@ function createEventsLog(events::T1; eventindex::Union{UnitRange, Nothing}=nothi subject = event[:subject] actioninput = event[:actioninput] outcome = event[:outcome] - str = "$subject: $actioninput $outcome" + str = "Action: $actioninput Outcome: $outcome" d = Dict{Symbol, String}(:name=>subject, :text=>str) push!(log, d) end @@ -348,6 +348,31 @@ function createEventsLog(events::T1; eventindex::Union{UnitRange, Nothing}=nothi end +function createChatLog(chatdict::T1; index::Union{UnitRange, Nothing}=nothing + ) where {T1<:AbstractVector} + # Initialize empty log array + log = Dict{Symbol, String}[] + + # Determine which indices to use - either provided range or full length + ind = + if index !== nothing + [index...] + else + 1:length(chatdict) + end + + # Iterate through events and format each one + for (i, event) in zip(ind, chatdict) + subject = event[:name] + text = event[:text] + d = Dict{Symbol, String}(:name=>subject, :text=>text) + push!(log, d) + end + + return log +end + + @@ -382,9 +407,6 @@ end - - -