1334 lines
45 KiB
Julia
1334 lines
45 KiB
Julia
module interface
|
||
|
||
export addNewMessage, conversation, decisionMaker, evaluator, reflector
|
||
# isterminal,
|
||
|
||
using JSON3, DataStructures, Dates, UUIDs, HTTP, Random, MQTTClient, PrettyPrinting
|
||
using GeneralUtils, LLMMCTS
|
||
using ..type, ..util, ..llmfunction
|
||
|
||
# ------------------------------------------------------------------------------------------------ #
|
||
# pythoncall setting #
|
||
# ------------------------------------------------------------------------------------------------ #
|
||
# Ref: https://github.com/JuliaPy/PythonCall.jl/issues/252
|
||
# by setting the following variables, PythonCall.jl will use:
|
||
# 1. system's python and packages installed by system (via apt install)
|
||
# or 2. conda python and packages installed by conda
|
||
# if these setting are not set (comment out), PythonCall will use its own python and packages that
|
||
# installed by CondaPkg.jl (from env_preparation.jl)
|
||
# ENV["JULIA_CONDAPKG_BACKEND"] = "Null" # set condapkg backend = none
|
||
# systemPython = split(read(`which python`, String), "\n")[1] # system's python path
|
||
# ENV["JULIA_PYTHONCALL_EXE"] = systemPython # find python location with $> which python ex. raw"/root/conda/bin/python"
|
||
|
||
# using PythonCall
|
||
# const py_agents = PythonCall.pynew()
|
||
# const py_llms = PythonCall.pynew()
|
||
# function __init__()
|
||
# # PythonCall.pycopy!(py_cv2, pyimport("cv2"))
|
||
|
||
# # equivalent to from urllib.request import urlopen in python
|
||
# PythonCall.pycopy!(py_agents, pyimport("langchain.agents"))
|
||
# PythonCall.pycopy!(py_llms, pyimport("langchain.llms"))
|
||
# end
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
|
||
macro executeStringFunction(functionStr, args...)
|
||
# Parse the function string into an expression
|
||
func_expr = Meta.parse(functionStr)
|
||
|
||
# Create a new function with the parsed expression
|
||
function_to_call = eval(Expr(:function,
|
||
Expr(:call, func_expr, args...), func_expr.args[2:end]...))
|
||
|
||
# Call the newly created function with the provided arguments
|
||
function_to_call(args...)
|
||
end
|
||
|
||
|
||
""" Think and choose action
|
||
|
||
# Arguments
|
||
- `config::T1`
|
||
config
|
||
- `state::T2`
|
||
a game state
|
||
|
||
# Return
|
||
- `thoughtDict::Dict`
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> config = Dict(
|
||
:mqttServerInfo => Dict(
|
||
:description => "mqtt server info",
|
||
:port => 1883,
|
||
:broker => "mqtt.yiem.cc"
|
||
),
|
||
:externalservice => Dict(
|
||
:text2textinstruct => Dict(
|
||
:mqtttopic => "/loadbalancer/requestingservice",
|
||
:description => "text to text service with instruct LLM",
|
||
:llminfo => Dict(
|
||
:name => "llama3instruct"
|
||
)
|
||
),
|
||
)
|
||
)
|
||
|
||
julia> output_thoughtDict = Dict(
|
||
:thought_1 => "The customer wants to buy a bottle of wine. This is a good start!",
|
||
:action_1 => Dict{Symbol, Any}(
|
||
:action=>"Chatbox",
|
||
:input=>"What occasion are you buying the wine for?"
|
||
),
|
||
:observation_1 => ""
|
||
)
|
||
```
|
||
|
||
# TODO
|
||
- [] update docstring
|
||
- [x] implement the function
|
||
- [] implement RAG to pull similar experience
|
||
- [] use customerinfo
|
||
- [] user storeinfo
|
||
- [x] add try block. check result that it is expected before returning
|
||
|
||
# Signature
|
||
"""
|
||
function decisionMaker(a::T)::Dict{Symbol, Any} where {T<:agent}
|
||
|
||
# lessonDict = copy(JSON3.read("lesson.json"))
|
||
|
||
# lesson =
|
||
# if isempty(lessonDict)
|
||
# ""
|
||
# else
|
||
# lessons = Dict{Symbol, Any}()
|
||
# for (k, v) in lessonDict
|
||
# lessons[k] = lessonDict[k][:lesson]
|
||
# end
|
||
|
||
# """
|
||
# You have attempted to help the user before and failed, either because your reasoning for the
|
||
# recommendation was incorrect or your response did not exactly match the user expectation.
|
||
# The following lesson(s) give a plan to avoid failing to help the user in the same way you
|
||
# did previously. Use them to improve your strategy to help the user.
|
||
|
||
# Here are some lessons in JSON format:
|
||
# $(JSON3.write(lessons))
|
||
|
||
# When providing the thought and action for the current trial, that into account these failed
|
||
# trajectories and make sure not to repeat the same mistakes and incorrect answers.
|
||
# """
|
||
# end
|
||
|
||
# _prompt =
|
||
# """
|
||
# You are a helpful sommelier working for a wine store.
|
||
# Your goal is to recommend the best wine from your inventory that match the user preferences.
|
||
# You are also keen to improve your recommendation with lesson(s).
|
||
|
||
# You must follow the following criteria:
|
||
# 1) Get to know how much the user willing to spend
|
||
# 2) Get to know type of wine the user is looking for e.g. red, white, sparkling, rose, dessert, fortified
|
||
# 3) Get to know what occasion the user is buying wine for
|
||
# 4) Get to know what characteristics of wine the user is looking for
|
||
# e.g. tannin, sweetness, intensity, acidity
|
||
# 5) Get to know what food the user will have with wine
|
||
# 6) Check your inventory for the best wine that match the user preference
|
||
# 7) Recommend wine to the user
|
||
|
||
# You should only respond with interleaving Thought, Action, Observation steps.
|
||
# Thought can reason about the current situation, and Action can be three types:
|
||
# 1) winestock[query], which you can use to find wine in your inventory. The more input data the better.
|
||
# 2) chatbox[text], which you can use to interact with the user.
|
||
# After each observation, provide the next Thought and next Action.
|
||
|
||
# You should only respond in JSON format as describe below:
|
||
# {
|
||
# "thought": "your reasoning",
|
||
# "action": {"name": "action to take", "input": "action input"},
|
||
# "observation": "result of the action"
|
||
# }
|
||
|
||
# Here are some examples:
|
||
# {
|
||
# "question": "I would like to buy a sedan with 8 seats.",
|
||
# "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.",
|
||
# "action_1": {"name": "inventory", "input": "sedan with 8 seats."},
|
||
# "observation_1": "Several model has 8 seats. Available color are black, red green"
|
||
# }
|
||
# {
|
||
# "thought": "I have a few color for the user to choose from. I will ask him what color he likes.",
|
||
# "action": {"name": "chatbox", "input": "Which color do you like?"}
|
||
# "observation": "I'll take black."
|
||
# }
|
||
|
||
# $lesson
|
||
|
||
# Let's begin!
|
||
|
||
# $(JSON3.write(state[:thoughtHistory]))
|
||
# {"thought"
|
||
# """
|
||
|
||
systemmsg =
|
||
"""
|
||
You are a helpful sommelier working for a wine store.
|
||
Your task is to help the user choose the best wine that match the user preferences from your inventory.
|
||
You are also eager to improve your helpfulness.
|
||
|
||
You must follow the following guidelines:
|
||
- Get to know how much the user willing to spend
|
||
- Get to know type of wine the user is looking for e.g. red, white, sparkling, rose, dessert, fortified
|
||
- Get to know what occasion the user is buying wine for
|
||
- Get to know what characteristics of wine the user is looking for e.g. tannin, sweetness, intensity, acidity
|
||
- Get to know what food the user will have with wine
|
||
|
||
At each round of conversation, the user will give you the current situation:
|
||
Context: ...
|
||
Your earlier conversation with the user: ...
|
||
|
||
You should then respond to the user with interleaving Thought, Plan, Action and Observation:
|
||
- thought:
|
||
1) State your reasoning about the current situation.
|
||
- plan: Based on the current situation, what would you do to complete the task? Be specific.
|
||
- action (Must be aligned with your plan): Can be one of the following functions:
|
||
1) CHATBOX[text], which you can use to talk with the user. "text" is in verbal English.
|
||
2) WINESTOCK[query], which you can use to find info about wine in your inventory. "query" is a search term in verbal English.
|
||
- observation: result of the action.
|
||
|
||
You should only respond in format as described below:
|
||
thought: ...
|
||
plan: ...
|
||
action_name: ...
|
||
action_input: ...
|
||
observation: ...
|
||
|
||
Let's begin!
|
||
"""
|
||
|
||
usermsg =
|
||
"""
|
||
Context: None
|
||
Your earlier conversation with the user: $(chatHistoryToString(a))
|
||
"""
|
||
|
||
_prompt =
|
||
[
|
||
Dict(:name=> "system", :text=> systemmsg),
|
||
Dict(:name=> "user", :text=> usermsg)
|
||
]
|
||
|
||
# put in model format
|
||
prompt = GeneralUtils.formatLLMtext(_prompt, "llama3instruct")
|
||
prompt *=
|
||
"""
|
||
<|start_header_id|>assistant<|end_header_id|>
|
||
"""
|
||
response = nothing # store for show when error msg show up
|
||
for attempt in 1:10
|
||
try
|
||
response = a.text2textInstructLLM(prompt)
|
||
responsedict = GeneralUtils.textToDict(response,
|
||
["thought", "plan", "action_name", "action_input", "observation"],
|
||
rightmarker=":", symbolkey=true)
|
||
|
||
if responsedict[:action_name] ∉ ["CHATBOX", "WINESTOCK"]
|
||
error("decisionMaker didn't use the given functions ", @__LINE__)
|
||
end
|
||
|
||
for i ∈ [:thought, :plan, :action_name]
|
||
if length(JSON3.write(responsedict[i])) == 0
|
||
error("$i is empty ", @__LINE__)
|
||
end
|
||
end
|
||
|
||
# check if there are more than 1 key per categories
|
||
for i ∈ [:thought, :plan, :action_name, :action_input, :observation]
|
||
matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
|
||
if length(matchkeys) > 1
|
||
error("DecisionMaker has more than one key per categories")
|
||
end
|
||
end
|
||
println("--> 1")
|
||
pprintln(responsedict)
|
||
return responsedict
|
||
catch e
|
||
io = IOBuffer()
|
||
showerror(io, e)
|
||
errorMsg = String(take!(io))
|
||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
println("")
|
||
println("Attempt $attempt. Error occurred: $errorMsg\n$st")
|
||
println("")
|
||
end
|
||
end
|
||
error("DecisionMaker failed to generate a thought ", response)
|
||
end
|
||
# function decisionMaker(a::T)::Dict{Symbol, Any} where {T<:agent}
|
||
|
||
# # lessonDict = copy(JSON3.read("lesson.json"))
|
||
|
||
# # lesson =
|
||
# # if isempty(lessonDict)
|
||
# # ""
|
||
# # else
|
||
# # lessons = Dict{Symbol, Any}()
|
||
# # for (k, v) in lessonDict
|
||
# # lessons[k] = lessonDict[k][:lesson]
|
||
# # end
|
||
|
||
# # """
|
||
# # You have attempted to help the user before and failed, either because your reasoning for the
|
||
# # recommendation was incorrect or your response did not exactly match the user expectation.
|
||
# # The following lesson(s) give a plan to avoid failing to help the user in the same way you
|
||
# # did previously. Use them to improve your strategy to help the user.
|
||
|
||
# # Here are some lessons in JSON format:
|
||
# # $(JSON3.write(lessons))
|
||
|
||
# # When providing the thought and action for the current trial, that into account these failed
|
||
# # trajectories and make sure not to repeat the same mistakes and incorrect answers.
|
||
# # """
|
||
# # end
|
||
|
||
# _prompt =
|
||
# """
|
||
# You are a helpful sommelier working for a wine store.
|
||
# Your goal is to recommend the best wine from your inventory that match the user preferences.
|
||
# You are also keen to improve your recommendation with lesson(s).
|
||
|
||
# You must follow the following criteria:
|
||
# 1) Get to know how much the user willing to spend
|
||
# 2) Get to know type of wine the user is looking for e.g. red, white, sparkling, rose, dessert, fortified
|
||
# 3) Get to know what occasion the user is buying wine for
|
||
# 4) Get to know what characteristics of wine the user is looking for
|
||
# e.g. tannin, sweetness, intensity, acidity
|
||
# 5) Get to know what food the user will have with wine
|
||
# 6) Check your inventory for the best wine that match the user preference
|
||
# 7) Recommend wine to the user
|
||
|
||
# You should only respond with interleaving Thought, Action, Observation steps.
|
||
# Thought can reason about the current situation, and Action can be three types:
|
||
# 1) winestock[query], which you can use to find wine in your inventory. The more input data the better.
|
||
# 2) chatbox[text], which you can use to interact with the user.
|
||
# After each observation, provide the next Thought and next Action.
|
||
|
||
# You should only respond in JSON format as describe below:
|
||
# {
|
||
# "thought": "your reasoning",
|
||
# "action": {"name": "action to take", "input": "action input"},
|
||
# "observation": "result of the action"
|
||
# }
|
||
|
||
# Here are some examples:
|
||
# {
|
||
# "question": "I would like to buy a sedan with 8 seats.",
|
||
# "thought_1": "Our showroom carries various vehicle model. But I'm not sure whether we have a models that fits the user demand, I need to check our inventory.",
|
||
# "action_1": {"name": "inventory", "input": "sedan with 8 seats."},
|
||
# "observation_1": "Several model has 8 seats. Available color are black, red green"
|
||
# }
|
||
# {
|
||
# "thought": "I have a few color for the user to choose from. I will ask him what color he likes.",
|
||
# "action": {"name": "chatbox", "input": "Which color do you like?"}
|
||
# "observation": "I'll take black."
|
||
# }
|
||
|
||
# $lesson
|
||
|
||
# Let's begin!
|
||
|
||
# $(JSON3.write(state[:thoughtHistory]))
|
||
# {"thought"
|
||
# """
|
||
|
||
# # apply LLM specific instruct format
|
||
# externalService = config[:externalservice][:text2textinstruct]
|
||
# llminfo = externalService[:llminfo]
|
||
# prompt =
|
||
# if llminfo[:name] == "llama3instruct"
|
||
# formatLLMtext_llama3instruct("system", _prompt)
|
||
# else
|
||
# error("llm model name is not defied yet $(@__LINE__)")
|
||
# end
|
||
|
||
# msgMeta = GeneralUtils.generate_msgMeta(
|
||
# externalService[:mqtttopic],
|
||
# senderName= "decisionMaker",
|
||
# senderId= string(uuid4()),
|
||
# receiverName= "text2textinstruct",
|
||
# mqttBroker= config[:mqttServerInfo][:broker],
|
||
# mqttBrokerPort= config[:mqttServerInfo][:port],
|
||
# )
|
||
|
||
# outgoingMsg = Dict(
|
||
# :msgMeta=> msgMeta,
|
||
# :payload=> Dict(
|
||
# :text=> prompt,
|
||
# :kwargs=> Dict(
|
||
# :max_tokens=> 512,
|
||
# :stop=> ["<|eot_id|>"],
|
||
# )
|
||
# )
|
||
# )
|
||
# @show outgoingMsg
|
||
|
||
# for attempt in 1:5
|
||
# try
|
||
# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
||
# _responseJsonStr = response[:response][:text]
|
||
# expectedJsonExample =
|
||
# """
|
||
# Here is an expected JSON format:
|
||
# {
|
||
# "thought": "...",
|
||
# "action": {"name": "...", "input": "..."},
|
||
# "observation": "..."
|
||
# }
|
||
# """
|
||
# responseJsonStr = jsoncorrection(config, _responseJsonStr, expectedJsonExample)
|
||
# thoughtDict = copy(JSON3.read(responseJsonStr))
|
||
|
||
# # check if dict has all required value
|
||
# thought::AbstractString = thoughtDict[:thought]
|
||
# actionname::AbstractString = thoughtDict[:action][:name]
|
||
# actioninput::AbstractString = thoughtDict[:action][:input]
|
||
# if actionname ∈ ["winestock", "chatbox", "recommendbox"]
|
||
# # LLM use available function
|
||
# elseif thought == ""
|
||
# error("DecisionMaker has no thought")
|
||
# elseif length(actioninput) == 0
|
||
# error("DecisionMaker has no actioninput")
|
||
# else
|
||
# error("DecisionMaker use wrong function")
|
||
# end
|
||
|
||
# return thoughtDict
|
||
# catch e
|
||
# io = IOBuffer()
|
||
# showerror(io, e)
|
||
# errorMsg = String(take!(io))
|
||
# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
# println("")
|
||
# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
|
||
# println("")
|
||
# end
|
||
# end
|
||
# error("DecisionMaker failed to generate a thought")
|
||
# end
|
||
|
||
|
||
""" Assigns a scalar value to each new child node to be used for selec-
|
||
tion and backpropagation. This value effectively quantifies the agent’s progress in task completion,
|
||
serving as a heuristic to steer the search algorithm towards the most promising regions of the tree.
|
||
|
||
# Arguments
|
||
- `a::T1`
|
||
one of Yiem's agent
|
||
- `state::T2`
|
||
a game state
|
||
|
||
# Return
|
||
- `evaluation::Tuple{String, Integer}`
|
||
evaluation and score
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia>
|
||
```
|
||
|
||
# Signature
|
||
"""
|
||
function evaluator(config::T1, state::T2
|
||
)::Tuple{String, Integer} where {T1<:AbstractDict, T2<:AbstractDict}
|
||
|
||
systemmsg =
|
||
"""
|
||
Analyze the trajectories of a solution to a question answering task. The trajectories are
|
||
labeled by environmental observations about the situation, thoughts that can reason about
|
||
the current situation and actions that can be three types:
|
||
1) winestock[query], which you can use to find wine in your inventory.
|
||
2) chatbox[text], which you can use to interact with the user.
|
||
3) recommendbox[answer], which returns your wine recommendation to the user.
|
||
|
||
Given a question and a trajectory, evaluate its correctness and provide your reasoning and
|
||
analysis in detail. Focus on the latest thought, action, and observation. Incomplete trajectories
|
||
can be correct if the thoughts and actions so far are correct, even if the answer is not found
|
||
yet. Do not generate additional thoughts or actions. Then ending with the correctness score s
|
||
where s is an integer from 0 to 10.
|
||
|
||
You should only respond in JSON format as describe below:
|
||
{"evaluation": "your evaluation", "score": "your evaluation score"}
|
||
|
||
Here are some examples:
|
||
user:
|
||
{
|
||
"question": "I'm looking for a sedan with an automatic driving feature.",
|
||
"thought_1": "I have many types of sedans in my inventory, each with diverse features.",
|
||
"thought_2": "But there is only 1 model that has the feature customer wanted.",
|
||
"thought_3": "I should check our inventory first to see if we have it.",
|
||
"action_1": {"name": "inventory", "input": "Yiem model A"},
|
||
"observation_1": "Yiem model A is in stock."
|
||
}
|
||
assistant
|
||
{
|
||
"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
|
||
It is also better to have simple searches corresponding to a single entity, making this the best action.",
|
||
"score": 10
|
||
}
|
||
|
||
user:
|
||
{
|
||
"question": "Do you have an all-in-one pen with 4 colors and a pencil for sale?",
|
||
"thought_1": "Let me check our inventory first to see if I have it.",
|
||
"action_1": {"name": "inventory", "input": "pen with 4 color and a pencil."},
|
||
"observation_1": "I found {1: "Pilot Dr. grip 4-in-1 pen", 2: "Rotting pencil"}",
|
||
"thought_2": "Ok, I have what the user is asking. Let's tell the user.",
|
||
"action_2": {"name": "chatbox", "input": "Yes, we do have a Pilot Dr. grip 4-in-1 pen and a Rotting pencil"},
|
||
"observation_1": "This is not what I wanted."
|
||
}
|
||
assistant:
|
||
{
|
||
"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
|
||
not a pen and a pencil seperately. A better search term should have been a 4-colors pen with a pencil, all-in-one.",
|
||
"score": 0
|
||
}
|
||
|
||
Let's begin!
|
||
"""
|
||
|
||
usermsg =
|
||
"""
|
||
$(JSON3.write(state[:thoughtHistory]))
|
||
"""
|
||
|
||
chathistory =
|
||
[
|
||
Dict(:name=> "system", :text=> systemmsg),
|
||
Dict(:name=> "user", :text=> usermsg)
|
||
]
|
||
|
||
# put in model format
|
||
prompt = formatLLMtext(chathistory, "llama3instruct")
|
||
prompt *=
|
||
"""
|
||
<|start_header_id|>assistant<|end_header_id|>
|
||
{
|
||
"""
|
||
|
||
pprint(prompt)
|
||
externalService = config[:externalservice][:text2textinstruct]
|
||
|
||
|
||
# apply LLM specific instruct format
|
||
externalService = config[:externalservice][:text2textinstruct]
|
||
|
||
msgMeta = GeneralUtils.generate_msgMeta(
|
||
externalService[:mqtttopic],
|
||
senderName= "evaluator",
|
||
senderId= string(uuid4()),
|
||
receiverName= "text2textinstruct",
|
||
mqttBroker= config[:mqttServerInfo][:broker],
|
||
mqttBrokerPort= config[:mqttServerInfo][:port],
|
||
)
|
||
|
||
outgoingMsg = Dict(
|
||
:msgMeta=> msgMeta,
|
||
:payload=> Dict(
|
||
:text=> prompt,
|
||
:kwargs=> Dict(
|
||
:max_tokens=> 512,
|
||
:stop=> ["<|eot_id|>"],
|
||
)
|
||
)
|
||
)
|
||
|
||
for attempt in 1:5
|
||
try
|
||
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
||
_responseJsonStr = response[:response][:text]
|
||
expectedJsonExample =
|
||
"""
|
||
Here is an expected JSON format:
|
||
{"evaluation": "...", "score": "..."}
|
||
"""
|
||
responseJsonStr = jsoncorrection(config, _responseJsonStr, expectedJsonExample)
|
||
evaluationDict = copy(JSON3.read(responseJsonStr))
|
||
|
||
# check if dict has all required value
|
||
dummya::AbstractString = evaluationDict[:evaluation]
|
||
dummyb::Integer = evaluationDict[:score]
|
||
|
||
return (evaluationDict[:evaluation], evaluationDict[:score])
|
||
catch e
|
||
io = IOBuffer()
|
||
showerror(io, e)
|
||
errorMsg = String(take!(io))
|
||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
println("")
|
||
@warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
|
||
println("")
|
||
end
|
||
end
|
||
error("evaluator failed to generate an evaluation")
|
||
end
|
||
|
||
|
||
"""
|
||
|
||
# Arguments
|
||
|
||
# Return
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia>
|
||
```
|
||
|
||
# TODO
|
||
- [] update docstring
|
||
- [x] implement the function
|
||
- [x] add try block. check result that it is expected before returning
|
||
|
||
# Signature
|
||
"""
|
||
function reflector(config::T1, state::T2)::String where {T1<:AbstractDict, T2<:AbstractDict}
|
||
# https://github.com/andyz245/LanguageAgentTreeSearch/blob/main/hotpot/hotpot.py
|
||
|
||
_prompt =
|
||
"""
|
||
You are a helpful sommelier working for a wine store.
|
||
Your goal is to recommend the best wine from your inventory that match the user preferences.
|
||
You will be given a question and a trajectory of the previous help you've done for a user.
|
||
You were unsuccessful in helping the user either because you guessed the wrong answer with Finish[answer], or you didn't know the user enough.
|
||
In a few sentences, Diagnose a possible reason for failure and devise a new, concise, high level plan that aims to mitigate the same failure.
|
||
Use complete sentences.
|
||
|
||
You should only respond in JSON format as describe below:
|
||
{"reflection": "your relection"}
|
||
|
||
Here are some examples:
|
||
Previous Trial:
|
||
{
|
||
"question": "Hello, I would like a get a bottle of wine",
|
||
"thought_1": "A customer wants to buy a bottle of wine. Before making a recommendation, I need to know more about their preferences.",
|
||
"action_1": {"name": "chatbox", "input": "What is the occasion for which you're buying this wine?"},
|
||
"observation_1": "We are holding a wedding party",
|
||
|
||
"thought_2": "A wedding party, that's a great occasion! The customer might be looking for a celebratory drink. Let me ask some more questions to narrow down the options.",
|
||
"action_2": {"name": "chatbox", "input": "What type of food will you be serving at the wedding?"},
|
||
"observation_2": "It will be Thai dishes.",
|
||
|
||
"thought_3": "With Thai food, I should recommend a wine that complements its spicy and savory flavors. And since it's a celebratory occasion, the customer might prefer a full-bodied wine.",
|
||
"action_3": {"name": "chatbox", "input": "What is your budget for this bottle of wine?"},
|
||
"observation_3": "I would spend up to 50 bucks.",
|
||
|
||
"thought_4": "Now that I have some more information, it's time to narrow down the options.",
|
||
"action_4": {"name": "winestock", "input": "red wine with full body, pairs well with spicy food, budget \$50"},
|
||
"observation_4": "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n",
|
||
|
||
"thought_5": "Now that I have a list of potential wines, I need to know more about the customer's taste preferences.",
|
||
"action_5": {"name": "chatbox", "input": "What type of wine characteristics are you looking for? (e.g. t.e.g. tannin level, sweetness, intensity, acidity)"},
|
||
"observation_5": "I like full-bodied red wine with low tannin.",
|
||
|
||
"thought_6": "Now that I have more information about the customer's preferences, it's time to make a recommendation.",
|
||
"action_6": {"name": "recommendbox", "input": "El Enemigo Cabernet Franc 2019"},
|
||
"observation_6": "I don't like the one you recommend. I want dry wine."
|
||
}
|
||
|
||
{
|
||
"reflection": "I asked the user about the occasion, food type, and budget, and then searched for wine in the inventory right away. However, I should have asked the user for the specific wine type and their preferences in order to gather more information before making a recommendation."
|
||
}
|
||
|
||
Let's begin!
|
||
|
||
Previous trial:
|
||
$(JSON3.write(state[:thoughtHistory]))
|
||
{"reflection"
|
||
"""
|
||
|
||
# apply LLM specific instruct format
|
||
externalService = config[:externalservice][:text2textinstruct]
|
||
llminfo = externalService[:llminfo]
|
||
prompt =
|
||
if llminfo[:name] == "llama3instruct"
|
||
formatLLMtext_llama3instruct("system", _prompt)
|
||
else
|
||
error("llm model name is not defied yet $(@__LINE__)")
|
||
end
|
||
|
||
msgMeta = GeneralUtils.generate_msgMeta(
|
||
a.config[:externalservice][:text2textinstruct][:mqtttopic],
|
||
senderName= "reflector",
|
||
senderId= string(uuid4()),
|
||
receiverName= "text2textinstruct",
|
||
mqttBroker= config[:mqttServerInfo][:broker],
|
||
mqttBrokerPort= config[:mqttServerInfo][:port],
|
||
)
|
||
|
||
outgoingMsg = Dict(
|
||
:msgMeta=> msgMeta,
|
||
:payload=> Dict(
|
||
:text=> prompt,
|
||
:kwargs=> Dict(
|
||
:max_tokens=> 512,
|
||
:stop=> ["<|eot_id|>"],
|
||
)
|
||
)
|
||
)
|
||
|
||
for attempt in 1:5
|
||
try
|
||
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
||
_responseJsonStr = response[:response][:text]
|
||
expectedJsonExample =
|
||
"""
|
||
Here is an expected JSON format:
|
||
{"reflection": "..."}
|
||
"""
|
||
responseJsonStr = jsoncorrection(config, _responseJsonStr, expectedJsonExample)
|
||
reflectionDict = copy(JSON3.read(responseJsonStr))
|
||
|
||
# check if dict has all required value
|
||
dummya::AbstractString = reflectionDict[:reflection]
|
||
|
||
return reflectionDict[:reflection]
|
||
catch e
|
||
io = IOBuffer()
|
||
showerror(io, e)
|
||
errorMsg = String(take!(io))
|
||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
println("")
|
||
@warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
|
||
println("")
|
||
end
|
||
end
|
||
error("reflector failed to generate a thought")
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
# """ Chat with llm.
|
||
|
||
# # Arguments
|
||
# `a::agent`
|
||
# an agent
|
||
|
||
# # Return
|
||
# None
|
||
|
||
# # Example
|
||
# ```jldoctest
|
||
# julia> using JSON3, UUIDs, Dates, FileIO, MQTTClient, ChatAgent
|
||
# julia> const mqttBroker = "mqtt.yiem.cc"
|
||
# julia> mqttclient, connection = MakeConnection(mqttBroker, 1883)
|
||
# julia> tools=Dict( # update input format
|
||
# "askbox"=>Dict(
|
||
# :description => "<askbox tool description>Useful for when you need to ask the user for more context. Do not ask the user their own question.</askbox tool description>",
|
||
# :input => "<input>Input is a text in JSON format.</input><input example>{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}</input example>",
|
||
# :output => "" ,
|
||
# :func => nothing,
|
||
# ),
|
||
# )
|
||
# julia> msgMeta = Dict(
|
||
# :msgPurpose=> "updateStatus",
|
||
# :from=> "agent",
|
||
# :to=> "llmAI",
|
||
# :requestresponse=> "request",
|
||
# :sendto=> "", # destination topic
|
||
# :replyTo=> "agent/api/v0.1.0/txt/response", # requester ask responseer to send reply to this topic
|
||
# :repondToMsgId=> "", # responseer is responseing to this msg id
|
||
# :taskstatus=> "", # "complete", "fail", "waiting" or other status
|
||
# :timestamp=> Dates.now(),
|
||
# :msgId=> "$(uuid4())",
|
||
# )
|
||
# julia> a = ChatAgent.agentReflex(
|
||
# "Jene",
|
||
# mqttclient,
|
||
# msgMeta,
|
||
# agentConfigTopic, # I need a function to send msg to config topic to get load balancer
|
||
# role=:sommelier,
|
||
# tools=tools
|
||
# )
|
||
# julia> newAgent = ChatAgent.agentReact(agent)
|
||
# julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?")
|
||
# ```
|
||
|
||
# # TODO
|
||
# - [] update docstring
|
||
# - [x] MCTS() for planning
|
||
# - [] add recap to initialState for earlier completed question
|
||
# - [WORKING] conversation loop
|
||
|
||
# # Signature
|
||
# """
|
||
# function conversation(a::T, userinput::Dict) where {T<:agent}
|
||
# config = deepcopy(a.config)
|
||
# pprint(config)
|
||
# if userinput[:text] == "newtopic"
|
||
# clearhistory(a)
|
||
# return "Okay. What shall we talk about?"
|
||
# else
|
||
# # add usermsg to a.chathistory
|
||
# addNewMessage(a, "user", userinput[:text])
|
||
|
||
# if isempty(a.plan[:currenttrajectory])
|
||
|
||
# # initial state
|
||
# a.plan[:currenttrajectory] = Dict{Symbol, Any}(
|
||
# # deepcopy the info to prevent modifying the info unintentionally during MCTS planning
|
||
# :customerinfo=> deepcopy(a.keywordinfo[:customerinfo]),
|
||
# :storeinfo=> deepcopy(a.keywordinfo[:storeinfo]),
|
||
# :userselect=> nothing,
|
||
# :reward=> 0,
|
||
# :isterminal=> false,
|
||
# :evaluation=> nothing,
|
||
# :lesson=> nothing,
|
||
|
||
# :totalTrajectoryReward=> nothing,
|
||
|
||
# # contain question, thought_1, action_1, observation_1, thought_2, ...
|
||
# :thoughtHistory=> OrderedDict{Symbol, Any}(
|
||
# #[] :recap=>,
|
||
# :question=> userinput[:text],
|
||
# ),
|
||
|
||
# # store conversation for virtual customer because the virtual customer agent is just
|
||
# # a function and stateless.
|
||
# :virtualCustomerChatHistory=> Vector{Dict{Symbol, Any}}(
|
||
# [Dict(:name=> "user", :text=> userinput[:text])]
|
||
# ),
|
||
# )
|
||
# else
|
||
# _, a.plan[:currenttrajectory] = makeNewState(a.plan[:currenttrajectory],
|
||
# a.plan[:activeplan][:thoughtHistory], userinput[:text], userinput[:select],
|
||
# userinput[:reward], userinput[:isterminal])
|
||
# end
|
||
# end
|
||
|
||
# while true
|
||
# bestNextState, besttrajectory = LLMMCTS.runMCTS(a.plan[:currenttrajectory],
|
||
# transition, config, decisionMaker, evaluator, reflector;
|
||
# totalsample=2, maxDepth=3, maxiterations=3, explorationweight=1.0)
|
||
# a.plan[:activeplan] = bestNextState
|
||
|
||
# latestActionKey, latestActionIndice =
|
||
# GeneralUtils.findHighestIndexKey(bestNextState[:thoughtHistory], "action")
|
||
# actionname = bestNextState[:thoughtHistory][latestActionKey][:name]
|
||
# actioninput = bestNextState[:thoughtHistory][latestActionKey][:input]
|
||
|
||
# # transition
|
||
# if actionname == "chatbox"
|
||
# # add usermsg to a.chathistory
|
||
# addNewMessage(a, "assistant", actioninput)
|
||
# return actioninput
|
||
# elseif actionname == "recommendbox"
|
||
# # add usermsg to a.chathistory
|
||
# addNewMessage(a, "assistant", actioninput)
|
||
# return actioninput
|
||
# else
|
||
# _, a.plan[:currenttrajectory] = transition(a, a.plan[:currenttrajectory], a.plan[:activeplan])
|
||
# end
|
||
# end
|
||
# end
|
||
|
||
|
||
|
||
""" Chat with llm.
|
||
|
||
# Arguments
|
||
`a::agent`
|
||
an agent
|
||
|
||
# Return
|
||
None
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> using JSON3, UUIDs, Dates, FileIO, MQTTClient, ChatAgent
|
||
julia> const mqttBroker = "mqtt.yiem.cc"
|
||
julia> mqttclient, connection = MakeConnection(mqttBroker, 1883)
|
||
julia> tools=Dict( # update input format
|
||
"askbox"=>Dict(
|
||
:description => "<askbox tool description>Useful for when you need to ask the user for more context. Do not ask the user their own question.</askbox tool description>",
|
||
:input => "<input>Input is a text in JSON format.</input><input example>{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}</input example>",
|
||
:output => "" ,
|
||
:func => nothing,
|
||
),
|
||
)
|
||
julia> msgMeta = Dict(
|
||
:msgPurpose=> "updateStatus",
|
||
:from=> "agent",
|
||
:to=> "llmAI",
|
||
:requestresponse=> "request",
|
||
:sendto=> "", # destination topic
|
||
:replyTo=> "agent/api/v0.1.0/txt/response", # requester ask responseer to send reply to this topic
|
||
:repondToMsgId=> "", # responseer is responseing to this msg id
|
||
:taskstatus=> "", # "complete", "fail", "waiting" or other status
|
||
:timestamp=> Dates.now(),
|
||
:msgId=> "$(uuid4())",
|
||
)
|
||
julia> a = ChatAgent.agentReflex(
|
||
"Jene",
|
||
mqttclient,
|
||
msgMeta,
|
||
agentConfigTopic, # I need a function to send msg to config topic to get load balancer
|
||
role=:sommelier,
|
||
tools=tools
|
||
)
|
||
julia> newAgent = ChatAgent.agentReact(agent)
|
||
julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?")
|
||
```
|
||
|
||
# TODO
|
||
- [] update docstring
|
||
- [x] MCTS() for planning
|
||
- [] add recap to initialState for earlier completed question
|
||
- [WORKING] conversation loop
|
||
|
||
# Signature
|
||
"""
|
||
function conversation(a::T, userinput::Dict) where {T<:agent}
|
||
|
||
if userinput[:text] == "newtopic"
|
||
clearhistory(a)
|
||
return "Okay. What shall we talk about?"
|
||
else
|
||
# add usermsg to a.chathistory
|
||
addNewMessage(a, "user", userinput[:text])
|
||
|
||
thought = think(a)
|
||
|
||
# thought will be added to chat model via context
|
||
chatresponse = generatechat(a, thought)
|
||
|
||
return chatresponse
|
||
end
|
||
|
||
|
||
end
|
||
# function conversation(a::T, userinput::Dict) where {T<:agent}
|
||
# config = deepcopy(a.config)
|
||
# pprint(config)
|
||
# if userinput[:text] == "newtopic"
|
||
# clearhistory(a)
|
||
# return "Okay. What shall we talk about?"
|
||
# else
|
||
# # add usermsg to a.chathistory
|
||
# addNewMessage(a, "user", userinput[:text])
|
||
|
||
# if isempty(a.plan[:currenttrajectory])
|
||
|
||
# # initial state
|
||
# a.plan[:currenttrajectory] = Dict{Symbol, Any}(
|
||
# # deepcopy the info to prevent modifying the info unintentionally during MCTS planning
|
||
# :customerinfo=> deepcopy(a.keywordinfo[:customerinfo]),
|
||
# :storeinfo=> deepcopy(a.keywordinfo[:storeinfo]),
|
||
# :userselect=> nothing,
|
||
# :reward=> 0,
|
||
# :isterminal=> false,
|
||
# :evaluation=> nothing,
|
||
# :lesson=> nothing,
|
||
|
||
# :totalTrajectoryReward=> nothing,
|
||
|
||
# # contain question, thought_1, action_1, observation_1, thought_2, ...
|
||
# :thoughtHistory=> OrderedDict{Symbol, Any}(
|
||
# #[] :recap=>,
|
||
# :question=> userinput[:text],
|
||
# ),
|
||
|
||
# # store conversation for virtual customer because the virtual customer agent is just
|
||
# # a function and stateless.
|
||
# :virtualCustomerChatHistory=> Vector{Dict{Symbol, Any}}(
|
||
# [Dict(:name=> "user", :text=> userinput[:text])]
|
||
# ),
|
||
# )
|
||
# else
|
||
# _, a.plan[:currenttrajectory] = makeNewState(a.plan[:currenttrajectory],
|
||
# a.plan[:activeplan][:thoughtHistory], userinput[:text], userinput[:select],
|
||
# userinput[:reward], userinput[:isterminal])
|
||
# end
|
||
# end
|
||
|
||
# while true
|
||
# bestNextState, besttrajectory = LLMMCTS.runMCTS(a.plan[:currenttrajectory],
|
||
# transition, config, decisionMaker, evaluator, reflector;
|
||
# totalsample=2, maxDepth=3, maxiterations=3, explorationweight=1.0)
|
||
# a.plan[:activeplan] = bestNextState
|
||
|
||
# latestActionKey, latestActionIndice =
|
||
# GeneralUtils.findHighestIndexKey(bestNextState[:thoughtHistory], "action")
|
||
# actionname = bestNextState[:thoughtHistory][latestActionKey][:name]
|
||
# actioninput = bestNextState[:thoughtHistory][latestActionKey][:input]
|
||
|
||
# # transition
|
||
# if actionname == "chatbox"
|
||
# # add usermsg to a.chathistory
|
||
# addNewMessage(a, "assistant", actioninput)
|
||
# return actioninput
|
||
# elseif actionname == "recommendbox"
|
||
# # add usermsg to a.chathistory
|
||
# addNewMessage(a, "assistant", actioninput)
|
||
# return actioninput
|
||
# else
|
||
# _, a.plan[:currenttrajectory] = transition(a, a.plan[:currenttrajectory], a.plan[:activeplan])
|
||
# end
|
||
# end
|
||
# end
|
||
|
||
|
||
"""
|
||
|
||
# Arguments
|
||
|
||
# Return
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia>
|
||
```
|
||
|
||
# TODO
|
||
- [] update docstring
|
||
- [x] implement the function
|
||
- [x] add try block. check result that it is expected before returning
|
||
|
||
# Signature
|
||
"""
|
||
function think(a::T) where {T<:agent}
|
||
thoughtDict = decisionMaker(a)
|
||
actionname = thoughtDict[:action_name]
|
||
actioninput = thoughtDict[:action_input]
|
||
|
||
# map action and input() to llm function
|
||
response =
|
||
if actionname == "CHATBOX"
|
||
(result=actioninput, errormsg=nothing, success=true)
|
||
elseif actionname == "WINESTOCK"
|
||
DBconnection = LibPQ.Connection("host=192.168.88.12 port=5432 dbname=yiem_wine_assistant user=yiem password=yiem@Postgres_0.0")
|
||
winestock(actioninput, DBconnection, a.text2textInstructLLM)
|
||
else
|
||
error("undefined LLM function. Requesting $actionname")
|
||
end
|
||
|
||
# this section allow LLM functions above to have different return values.
|
||
result = haskey(response, :result) ? response[:result] : nothing
|
||
select = haskey(response, :select) ? response[:select] : nothing
|
||
reward::Integer = haskey(response, :reward) ? response[:reward] : 0
|
||
isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false
|
||
errormsg::Union{AbstractString, Nothing} = haskey(response, :errormsg) ? response[:errormsg] : nothing
|
||
success::Bool = haskey(response, :success) ? response[:success] : false
|
||
a.shortmem
|
||
return result
|
||
end
|
||
|
||
|
||
"""
|
||
|
||
# Arguments
|
||
- `a::T1`
|
||
one of ChatAgent's agent.
|
||
- `input::T2`
|
||
# Return
|
||
A JSON string of available wine
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia>
|
||
```
|
||
|
||
# TODO
|
||
- [] update docs
|
||
- [WORKING] implement the function
|
||
|
||
# Signature
|
||
"""
|
||
function generatechat(a::T1, input::T2) where {T1<:agent, T2<:AbstractString}
|
||
systemmsg =
|
||
"""
|
||
You are a helpful sommelier working for a wine store.
|
||
Your task is to help the user choose the best wine that match the user preferences from your inventory.
|
||
You are also eager to improve your helpfulness.
|
||
|
||
You must follow the following guidelines:
|
||
- Get to know how much the user willing to spend
|
||
- Get to know type of wine the user is looking for e.g. red, white, sparkling, rose, dessert, fortified
|
||
- Get to know what occasion the user is buying wine for
|
||
- Get to know what characteristics of wine the user is looking for e.g. tannin, sweetness, intensity, acidity
|
||
- Get to know what food the user will have with wine
|
||
|
||
At each round of conversation, the user will give you:
|
||
Context: ...
|
||
Your thoughts: Your current thinking in your mind
|
||
Your earlier conversation with the user: ...
|
||
|
||
You should then respond to the user with:
|
||
- chat: what do you want to say to the user
|
||
|
||
You should only respond in format as described below:
|
||
chat: ...
|
||
|
||
Let's begin!
|
||
"""
|
||
|
||
usermsg =
|
||
"""
|
||
Context: None
|
||
Your thoughts: $input
|
||
Your earlier conversation with the user: $(chatHistoryToString(a))
|
||
"""
|
||
|
||
_prompt =
|
||
[
|
||
Dict(:name=> "system", :text=> systemmsg),
|
||
Dict(:name=> "user", :text=> usermsg)
|
||
]
|
||
|
||
# put in model format
|
||
prompt = GeneralUtils.formatLLMtext(_prompt, "llama3instruct")
|
||
prompt *=
|
||
"""
|
||
<|start_header_id|>assistant<|end_header_id|>
|
||
"""
|
||
|
||
for attempt in 1:5
|
||
try
|
||
response = text2textInstructLLM(prompt)
|
||
responsedict = GeneralUtils.textToDict(response,
|
||
["chat"],
|
||
rightmarker=":", symbolkey=true)
|
||
|
||
# check if dict has all required value
|
||
evaluationtext::AbstractString = responsedict[:evaluation]
|
||
responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5
|
||
score::Integer = responsedict[:score]
|
||
accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
|
||
suggestion::AbstractString = responsedict[:suggestion]
|
||
|
||
# add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension)
|
||
state[:evaluation] = responsedict[:evaluation]
|
||
state[:evaluationscore] = responsedict[:score]
|
||
state[:accepted_as_answer] = responsedict[:accepted_as_answer]
|
||
state[:suggestion] = responsedict[:suggestion]
|
||
|
||
# mark as terminal state when the answer is achieved
|
||
if accepted_as_answer == "Yes"
|
||
state[:isterminal] = true
|
||
state[:reward] = 1
|
||
end
|
||
|
||
return responsedict[:score]
|
||
catch e
|
||
io = IOBuffer()
|
||
showerror(io, e)
|
||
errorMsg = String(take!(io))
|
||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
println("")
|
||
println("Attempt $attempt. Error occurred: $errorMsg\n$st")
|
||
println("")
|
||
end
|
||
end
|
||
error("evaluator failed to generate an evaluation")
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
end
|
||
|
||
|
||
|
||
# """
|
||
|
||
# # Arguments
|
||
# - `a::T1`
|
||
# one of Yiem's agent
|
||
# - `state::T2`
|
||
# a game state
|
||
|
||
# # Return
|
||
# - `evaluation::Tuple{String, Integer}`
|
||
# evaluation and score
|
||
|
||
# # Example
|
||
# ```jldoctest
|
||
# julia>
|
||
# ```
|
||
|
||
# # TODO
|
||
# - [] update docs
|
||
# - [] implement the function
|
||
|
||
# # Signature
|
||
# """
|
||
# function comparer(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T2<:AbstractDict}
|
||
|
||
# _prompt =
|
||
# """
|
||
# Analyze the trajectories of a solution to a question answering task. The trajectories are
|
||
# labeled by environmental observations about the situation, thoughts that can reason about
|
||
# the current situation and actions that can be three types:
|
||
# 1) winestock[query], which you can use to find wine in your inventory.
|
||
# 2) chatbox[text], which you can use to interact with the user.
|
||
# 3) recommendbox[answer], which returns your wine recommendation to the user.
|
||
|
||
# Given a question and a trajectory, evaluate its correctness and provide your reasoning and
|
||
# analysis in detail. Focus on the latest thought, action, and observation. Incomplete trajectories
|
||
# can be correct if the thoughts and actions so far are correct, even if the answer is not found
|
||
# yet. Do not generate additional thoughts or actions. Then ending with the correctness score s
|
||
# where s is an integer from 0 to 10.
|
||
|
||
# You should only respond in JSON format as describe below:
|
||
# {"evaluation": "your evaluation", "score": "your evaluation score"}
|
||
|
||
# Here are some examples:
|
||
# {
|
||
# "question": "I'm looking for a sedan with an automatic driving feature.",
|
||
# "thought_1": "I have many types of sedans in my inventory, each with diverse features.",
|
||
# "thought_2": "But there is only 1 model that has the feature customer wanted.",
|
||
# "thought_3": "I should check our inventory first to see if we have it.",
|
||
# "action_1": {"name": "inventory", "input": "Yiem model A"},
|
||
# "observation_1": "Yiem model A is in stock."
|
||
# }
|
||
# {"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
|
||
# It is also better to have simple searches corresponding to a single entity, making this the best action.",
|
||
# "score": 10
|
||
# }
|
||
|
||
# {
|
||
# "question": "Do you have an all-in-one pen with 4 colors and a pencil for sale?",
|
||
# "thought_1": "Let me check our inventory first to see if I have it.",
|
||
# "action_1": {"name": "inventory", "input": "pen with 4 color and a pencil."},
|
||
# "observation_1": "I found {1: "Pilot Dr. grip 4-in-1 pen", 2: "Rotting pencil"}",
|
||
# "thought_2": "Ok, I have what the user is asking. Let's tell the user.",
|
||
# "action_2": {"name": "chatbox", "input": "Yes, we do have a Pilot Dr. grip 4-in-1 pen and a Rotting pencil"},
|
||
# "observation_1": "This is not what I wanted."
|
||
# }
|
||
# {"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
|
||
# not a pen and a pencil seperately. A better search term should have been a 4-colors pen with a pencil, all-in-one.",
|
||
# "score": 0
|
||
# }
|
||
|
||
# Let's begin!:
|
||
# $(JSON3.write(state[:thoughtHistory]))
|
||
# {"evaluation"
|
||
# """
|
||
|
||
# # apply LLM specific instruct format
|
||
# externalService = a.config[:externalservice][:text2textinstruct]
|
||
# llminfo = externalService[:llminfo]
|
||
# prompt =
|
||
# if llminfo[:name] == "llama3instruct"
|
||
# formatLLMtext_llama3instruct("system", _prompt)
|
||
# else
|
||
# error("llm model name is not defied yet $(@__LINE__)")
|
||
# end
|
||
|
||
# msgMeta = GeneralUtils.generate_msgMeta(
|
||
# a.config[:externalservice][:text2textinstruct][:mqtttopic],
|
||
# senderName= "evaluator",
|
||
# senderId= a.id,
|
||
# receiverName= "text2textinstruct",
|
||
# mqttBroker= a.config[:mqttServerInfo][:broker],
|
||
# mqttBrokerPort= a.config[:mqttServerInfo][:port],
|
||
# )
|
||
|
||
# outgoingMsg = Dict(
|
||
# :msgMeta=> msgMeta,
|
||
# :payload=> Dict(
|
||
# :text=> prompt,
|
||
# :kwargs=> Dict(
|
||
# :max_tokens=> 512,
|
||
# :stop=> ["<|eot_id|>"],
|
||
# )
|
||
# )
|
||
# )
|
||
|
||
# for attempt in 1:5
|
||
# try
|
||
# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
|
||
# _responseJsonStr = response[:response][:text]
|
||
# expectedJsonExample =
|
||
# """
|
||
# Here is an expected JSON format:
|
||
# {"evaluation": "...", "score": "..."}
|
||
# """
|
||
# responseJsonStr = jsoncorrection(a, _responseJsonStr, expectedJsonExample)
|
||
# evaluationDict = copy(JSON3.read(responseJsonStr))
|
||
|
||
# # check if dict has all required value
|
||
# dummya::AbstractString = evaluationDict[:evaluation]
|
||
# dummyb::Integer = evaluationDict[:score]
|
||
|
||
# return (evaluationDict[:evaluation], evaluationDict[:score])
|
||
# catch e
|
||
# io = IOBuffer()
|
||
# showerror(io, e)
|
||
# errorMsg = String(take!(io))
|
||
# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
# println("")
|
||
# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
|
||
# println("")
|
||
# end
|
||
# end
|
||
# error("evaluator failed to generate an evaluation")
|
||
# end
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
end # module interface |