ton/YiemAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat lamaiin
2024-05-31 11:47:43 +07:00
parent 3f38fdbb70
commit 3196842296
17 changed files with 5607 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
src/interface.jl
View File

@@ -4,7 +4,7 @@ export addNewMessage, conversation, decisionMaker, evaluator, reflector
# isterminal,
using JSON3, DataStructures, Dates, UUIDs, HTTP, Random, MQTTClient, PrettyPrinting
using GeneralUtils
using GeneralUtils, LLMMCTS
using ..type, ..util, ..llmfunction, ..mcts
# ------------------------------------------------------------------------------------------------ #
@@ -267,7 +267,7 @@ julia>
"""
function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T2<:AbstractDict}
_prompt =
systemmsg =
"""
Analyze the trajectories of a solution to a question answering task. The trajectories are
labeled by environmental observations about the situation, thoughts that can reason about
@@ -286,6 +286,7 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
{"evaluation": "your evaluation", "score": "your evaluation score"}
Here are some examples:
user:
{
"question": "I'm looking for a sedan with an automatic driving feature.",
"thought_1": "I have many types of sedans in my inventory, each with diverse features.",
@@ -294,11 +295,14 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
"action_1": {"name": "inventory", "input": "Yiem model A"},
"observation_1": "Yiem model A is in stock."
}
{"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
assistant
{
"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
It is also better to have simple searches corresponding to a single entity, making this the best action.",
"score": 10
}
user:
{
"question": "Do you have an all-in-one pen with 4 colors and a pencil for sale?",
"thought_1": "Let me check our inventory first to see if I have it.",
@@ -308,28 +312,44 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
"action_2": {"name": "chatbox", "input": "Yes, we do have a Pilot Dr. grip 4-in-1 pen and a Rotting pencil"},
"observation_1": "This is not what I wanted."
}
{"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
assistant:
{
"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
not a pen and a pencil seperately. A better search term should have been a 4-colors pen with a pencil, all-in-one.",
"score": 0
}
Let's begin!:
$(JSON3.write(state[:thoughtHistory]))
{"evaluation"
Let's begin!
"""
usermsg =
"""
$(JSON3.write(state[:thoughtHistory]))
"""
chathistory =
[
Dict(:name=> "system", :text=> systemmsg),
Dict(:name=> "user", :text=> usermsg)
]
# put in model format
prompt = formatLLMtext(chathistory, "llama3instruct")
prompt *=
"""
<|start_header_id|>assistant<|end_header_id|>
{
"""
pprint(prompt)
externalService = a.config[:externalservice][:text2textinstruct]
# apply LLM specific instruct format
externalService = a.config[:externalservice][:text2textinstruct]
llminfo = externalService[:llminfo]
prompt =
if llminfo[:name] == "llama3instruct"
formatLLMtext_llama3instruct("system", _prompt)
else
error("llm model name is not defied yet $(@__LINE__)")
end
msgMeta = GeneralUtils.generate_msgMeta(
a.config[:externalservice][:text2textinstruct][:mqtttopic],
externalService[:mqtttopic],
senderName= "evaluator",
senderId= a.id,
receiverName= "text2textinstruct",
@@ -377,6 +397,123 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
end
error("evaluator failed to generate an evaluation")
end
# function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T2<:AbstractDict}
# _prompt =
# """
# Analyze the trajectories of a solution to a question answering task. The trajectories are
# labeled by environmental observations about the situation, thoughts that can reason about
# the current situation and actions that can be three types:
# 1) winestock[query], which you can use to find wine in your inventory.
# 2) chatbox[text], which you can use to interact with the user.
# 3) recommendbox[answer], which returns your wine recommendation to the user.
# Given a question and a trajectory, evaluate its correctness and provide your reasoning and
# analysis in detail. Focus on the latest thought, action, and observation. Incomplete trajectories
# can be correct if the thoughts and actions so far are correct, even if the answer is not found
# yet. Do not generate additional thoughts or actions. Then ending with the correctness score s
# where s is an integer from 0 to 10.
# You should only respond in JSON format as describe below:
# {"evaluation": "your evaluation", "score": "your evaluation score"}
# Here are some examples:
# {
# "question": "I'm looking for a sedan with an automatic driving feature.",
# "thought_1": "I have many types of sedans in my inventory, each with diverse features.",
# "thought_2": "But there is only 1 model that has the feature customer wanted.",
# "thought_3": "I should check our inventory first to see if we have it.",
# "action_1": {"name": "inventory", "input": "Yiem model A"},
# "observation_1": "Yiem model A is in stock."
# }
# {"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
# It is also better to have simple searches corresponding to a single entity, making this the best action.",
# "score": 10
# }
# {
# "question": "Do you have an all-in-one pen with 4 colors and a pencil for sale?",
# "thought_1": "Let me check our inventory first to see if I have it.",
# "action_1": {"name": "inventory", "input": "pen with 4 color and a pencil."},
# "observation_1": "I found {1: "Pilot Dr. grip 4-in-1 pen", 2: "Rotting pencil"}",
# "thought_2": "Ok, I have what the user is asking. Let's tell the user.",
# "action_2": {"name": "chatbox", "input": "Yes, we do have a Pilot Dr. grip 4-in-1 pen and a Rotting pencil"},
# "observation_1": "This is not what I wanted."
# }
# {"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
# not a pen and a pencil seperately. A better search term should have been a 4-colors pen with a pencil, all-in-one.",
# "score": 0
# }
# Let's begin!:
# $(JSON3.write(state[:thoughtHistory]))
# {"evaluation"
# """
# # apply LLM specific instruct format
# externalService = a.config[:externalservice][:text2textinstruct]
# llminfo = externalService[:llminfo]
# prompt =
# if llminfo[:name] == "llama3instruct"
# formatLLMtext_llama3instruct("system", _prompt)
# else
# error("llm model name is not defied yet $(@__LINE__)")
# end
# msgMeta = GeneralUtils.generate_msgMeta(
# a.config[:externalservice][:text2textinstruct][:mqtttopic],
# senderName= "evaluator",
# senderId= a.id,
# receiverName= "text2textinstruct",
# mqttBroker= a.config[:mqttServerInfo][:broker],
# mqttBrokerPort= a.config[:mqttServerInfo][:port],
# )
# outgoingMsg = Dict(
# :msgMeta=> msgMeta,
# :payload=> Dict(
# :text=> prompt,
# :kwargs=> Dict(
# :max_tokens=> 512,
# :stop=> ["<|eot_id|>"],
# )
# )
# )
# for attempt in 1:5
# try
# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
# _responseJsonStr = response[:response][:text]
# expectedJsonExample =
# """
# Here is an expected JSON format:
# {"evaluation": "...", "score": "..."}
# """
# responseJsonStr = jsoncorrection(a, _responseJsonStr, expectedJsonExample)
# evaluationDict = copy(JSON3.read(responseJsonStr))
# # check if dict has all required value
# dummya::AbstractString = evaluationDict[:evaluation]
# dummyb::Integer = evaluationDict[:score]
# return (evaluationDict[:evaluation], evaluationDict[:score])
# catch e
# io = IOBuffer()
# showerror(io, e)
# errorMsg = String(take!(io))
# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
# println("")
# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
# println("")
# end
# end
# error("evaluator failed to generate an evaluation")
# end
# """
@@ -784,7 +921,7 @@ function conversation(a::T, userinput::Dict) where {T<:agent}
while true
bestNextState, besttrajectory = runMCTS(a, a.plan[:currenttrajectory], decisionMaker,
evaluator, reflector, totalsample=2, maxDepth=2, maxiterations=2, explorationweight=1.0)
evaluator, reflector, totalsample=2, maxDepth=3, maxiterations=3, explorationweight=1.0)
a.plan[:activeplan] = bestNextState
latestActionKey, latestActionIndice =

453
src/llmfunction.jl
View File

@@ -365,58 +365,86 @@ julia> result = winestock(agent, input)
function winestock(a::T1, input::T2
)::Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}} where {T1<:agent, T2<:AbstractString}
# SELECT *
# FROM food
# WHERE 'China' = ANY(food_name)
# OR 'India' = ANY(food_name);
wineattributes = wineattributes_wordToNumber(a, input)
systemmsg =
"""
As an attentive sommelier, your mission is to determine the user's preferred levels of sweetness, intensity, tannin, and acidity for a wine based on their input.
You'll achieve this by referring to the provided conversion table.
Conversion Table:
Intensity level:
Level 1: May correspond to "light-bodied" or a similar description.
Level 2: May correspond to "med-light" or a similar description.
Level 3: May correspond to "medium" or a similar description.
Level 4: May correspond to "med-full" or a similar description.
Level 5: May correspond to "full" or a similar description.
Sweetness level:
Level 1: May correspond to "dry", "no-sweet" or a similar description.
Level 2: May correspond to "off-dry", "less-sweet" or a similar description.
Level 3: May correspond to "semi-sweet" or a similar description.
Level 4: May correspond to "sweet" or a similar description.
Level 5: May correspond to "very sweet" or a similar description.
Tannin level:
Level 1: May correspond to "low tannin" or a similar description.
Level 2: May correspond to "semi-low tannin" or a similar description.
Level 3: May correspond to "medium tannin" or a similar description.
Level 4: May correspond to "semi-high tannin" or a similar description.
Level 5: May correspond to "high tannin" or a similar description.
Acidity level:
Level 1: May correspond to "low acidity" or a similar description.
Level 2: May correspond to "semi-low acidity" or a similar description.
Level 3: May correspond to "medium acidity" or a similar description.
Level 4: May correspond to "semi-high acidity" or a similar description.
Level 5: May correspond to "high acidity" or a similar description.
As a helpful sommelier, your mission is to write SQL queries that search the PostgreSQL database for wines based on user input.
You should only respond in JSON format as describe below:
{
The database has the following tables (schema):
1. Table wine (
wine_id uuid primary key,
wine_name varchar,
brand varchar,
manufacturer varchar,
region varchar,
country varchar,
wine_type varchar,
grape_variety varchar,
serving_temperature varchar,
intensity integer,
sweetness integer,
tannin integer,
acidity integer,
fizziness integer,
other_attributes jsonb,
created_at timestamptz,
updated_at timestamptz,
description text
)
2. Table food (
food_id uuid primary key,
food_name varchar,
country varchar,
spicy integer,
sweet integer,
sour integer,
umami integer,
bitter integer,
serving_temperature integer,
other_attributes jsonb,
created_at timestamptz,
updated_at timestamptz,
description text
)
3. wine_food (
wine_id uuid references wine(wine_id),
food_id uuid references food(food_id),
constraint wine_food_id primary key (wine_id, food_id),
created_at timestamptz,
updated_at timestamptz
)
You should only respond in JSON format as describe below:
{
"SQL":
{
"sweetness": "sweetness level",
"acidity": "acidity level",
"tannin": "tannin level",
"intensity": "intensity level"
}
}
}
Here are some examples:
Here are some examples:
user: red wines, price < 50, body=full-bodied, tannins=1, off dry, acidity=medium, intensity=intense, Thai dishes
assistant:
{
"wine_attributes":
{
"sweetness": 2,
"acidity": 3,
"tannin": 1,
"intensity": 5
}
}
user: {"sweetness": 2,"acidity": 3,"tannin": 1,"intensity": 5, "food": "Thai"}
assistant:
{
"SQL":
}
Let's begin!
"""
@@ -440,107 +468,246 @@ function winestock(a::T1, input::T2
{
"""
pprint(prompt)
externalService = a.config[:externalservice][:text2textinstruct]
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
msgMeta = GeneralUtils.generate_msgMeta(
externalService[:mqtttopic],
senderName= "virtualWineUserChatbox",
senderId= a.id,
receiverName= "text2textinstruct",
mqttBroker= a.config[:mqttServerInfo][:broker],
mqttBrokerPort= a.config[:mqttServerInfo][:port],
msgId = "dummyid" #CHANGE remove after testing finished
)
outgoingMsg = Dict(
:msgMeta=> msgMeta,
:payload=> Dict(
:text=> prompt,
)
)
attempt = 0
for attempt in 1:5
try
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
_responseJsonStr = response[:response][:text]
expectedJsonExample =
"""
Here is an expected JSON format:
{
"wine_attributes":
{
"...": "...",
"...": "...",
}
}
"""
responseJsonStr = jsoncorrection(a, _responseJsonStr, expectedJsonExample)
responseDict = copy(JSON3.read(responseJsonStr))
return (text, select, reward, isterminal)
catch e
io = IOBuffer()
showerror(io, e)
errorMsg = String(take!(io))
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
println("")
@warn "Error occurred: $errorMsg\n$st"
println("")
end
end
error("virtualWineUserChatbox failed to get a response")
winesStr =
"""
1: El Enemigo Cabernet Franc 2019
2: Tantara Chardonnay 2017
"""
result =
"""
I found the following wines in our stock:
{
$winesStr
}
"""
return result, nothing, 0, false
end
# function winestock(a::T1, input::T2
# )::Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}} where {T1<:agent, T2<:AbstractString}
# winesStr =
# """
# 1: El Enemigo Cabernet Franc 2019
# 2: Tantara Chardonnay 2017
# """
# result =
# """
# I found the following wines in our stock:
# {
# $winesStr
# }
# """
# return result, nothing, 0, false
# end
pprint(prompt)
externalService = a.config[:externalservice][:text2textinstruct]
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
msgMeta = GeneralUtils.generate_msgMeta(
externalService[:mqtttopic],
senderName= "virtualWineUserChatbox",
senderId= a.id,
receiverName= "text2textinstruct",
mqttBroker= a.config[:mqttServerInfo][:broker],
mqttBrokerPort= a.config[:mqttServerInfo][:port],
msgId = "dummyid" #CHANGE remove after testing finished
)
outgoingMsg = Dict(
:msgMeta=> msgMeta,
:payload=> Dict(
:text=> prompt,
)
)
attempt = 0
for attempt in 1:5
try
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
_responseJsonStr = response[:response][:text]
expectedJsonExample =
"""
Here is an expected JSON format:
{
"attributes":
{
"...": "...",
"...": "...",
}
}
"""
responseJsonStr = jsoncorrection(a, _responseJsonStr, expectedJsonExample)
_responseDict = copy(JSON3.read(responseJsonStr))
responseDict = _responseDict[:attributes]
return responseDict
catch e
io = IOBuffer()
showerror(io, e)
errorMsg = String(take!(io))
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
println("")
@warn "Error occurred: $errorMsg\n$st"
println("")
end
end
error("wineattributes_wordToNumber() failed to get a response")
# winesStr =
# """
# 1: El Enemigo Cabernet Franc 2019
# 2: Tantara Chardonnay 2017
# """
# result =
# """
# I found the following wines in our stock:
# {
# $winesStr
# }
# """
# return result, nothing, 0, false
end
function wineattributes_wordToNumber(a::T1, input::T2
)::Dict where {T1<:agent, T2<:AbstractString}
systemmsg =
"""
As an attentive sommelier, your mission is to determine the user's preferred levels of sweetness, intensity, tannin, acidity and other criteria for a wine based on their input.
You'll achieve this by referring to the provided conversion table.
Conversion Table:
Intensity level:
Level 1: May correspond to "light-bodied" or a similar description.
Level 2: May correspond to "med light", "medium light" or a similar description.
Level 3: May correspond to "medium" or a similar description.
Level 4: May correspond to "med full", "medium full" or a similar description.
Level 5: May correspond to "full" or a similar description.
Sweetness level:
Level 1: May correspond to "dry", "no sweet" or a similar description.
Level 2: May correspond to "off dry", "less sweet" or a similar description.
Level 3: May correspond to "semi sweet" or a similar description.
Level 4: May correspond to "sweet" or a similar description.
Level 5: May correspond to "very sweet" or a similar description.
Tannin level:
Level 1: May correspond to "low tannin" or a similar description.
Level 2: May correspond to "semi low tannin" or a similar description.
Level 3: May correspond to "medium tannin" or a similar description.
Level 4: May correspond to "semi high tannin" or a similar description.
Level 5: May correspond to "high tannin" or a similar description.
Acidity level:
Level 1: May correspond to "low acidity" or a similar description.
Level 2: May correspond to "semi low acidity" or a similar description.
Level 3: May correspond to "medium acidity" or a similar description.
Level 4: May correspond to "semi high acidity" or a similar description.
Level 5: May correspond to "high acidity" or a similar description.
You should only respond in JSON format as describe below:
{
"attributes":
{
"sweetness": "sweetness level",
"acidity": "acidity level",
"tannin": "tannin level",
"intensity": "intensity level"
}
}
Here are some examples:
user: "price < 25, full-bodied white wine with sweetness level 2, low tannin level and medium acidity level, Pizza"
assistant:
{
"attributes":
{
"wine_type": "white"
"budget": less than 25",
"food_pairing": "Pizza",
"sweetness": 2,
"acidity": 3,
"tannin": 1,
"intensity": 5
}
}
user: body=full-bodied, off dry, acidity=medium, intensity=intense
assistant:
{
"attributes":
{
"sweetness": 2,
"acidity": 3,
"tannin": "not specified",
"intensity": 5
}
}
Let's begin!
"""
usermsg =
"""
$input
"""
chathistory =
[
Dict(:name=> "system", :text=> systemmsg),
Dict(:name=> "user", :text=> usermsg)
]
# put in model format
prompt = formatLLMtext(chathistory, "llama3instruct")
prompt *=
"""
<|start_header_id|>assistant<|end_header_id|>
{
"""
pprint(prompt)
externalService = a.config[:externalservice][:text2textinstruct]
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
msgMeta = GeneralUtils.generate_msgMeta(
externalService[:mqtttopic],
senderName= "wineattributes_wordToNumber",
senderId= a.id,
receiverName= "text2textinstruct",
mqttBroker= a.config[:mqttServerInfo][:broker],
mqttBrokerPort= a.config[:mqttServerInfo][:port],
)
outgoingMsg = Dict(
:msgMeta=> msgMeta,
:payload=> Dict(
:text=> prompt,
)
)
attempt = 0
for attempt in 1:5
try
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
_responseJsonStr = response[:response][:text]
expectedJsonExample =
"""
Here is an expected JSON format:
{
"attributes":
{
"...": "...",
"...": "...",
}
}
"""
responseJsonStr = jsoncorrection(a, _responseJsonStr, expectedJsonExample)
_responseDict = copy(JSON3.read(responseJsonStr))
responseDict = _responseDict[:attributes]
return responseDict
catch e
io = IOBuffer()
showerror(io, e)
errorMsg = String(take!(io))
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
println("")
@warn "Error occurred: $errorMsg\n$st"
println("")
end
end
error("wineattributes_wordToNumber() failed to get a response")
end
""" Attemp to correct LLM response's incorrect JSON response.