ton/YiemAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat lamaiin
2024-05-31 11:47:43 +07:00
parent 3f38fdbb70
commit 3196842296
17 changed files with 5607 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
src/interface.jl
View File

@@ -4,7 +4,7 @@ export addNewMessage, conversation, decisionMaker, evaluator, reflector
# isterminal,
using JSON3, DataStructures, Dates, UUIDs, HTTP, Random, MQTTClient, PrettyPrinting
using GeneralUtils
using GeneralUtils, LLMMCTS
using ..type, ..util, ..llmfunction, ..mcts
# ------------------------------------------------------------------------------------------------ #
@@ -267,7 +267,7 @@ julia>
"""
function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T2<:AbstractDict}
_prompt =
systemmsg =
"""
Analyze the trajectories of a solution to a question answering task. The trajectories are
labeled by environmental observations about the situation, thoughts that can reason about
@@ -286,6 +286,7 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
{"evaluation": "your evaluation", "score": "your evaluation score"}
Here are some examples:
user:
{
"question": "I'm looking for a sedan with an automatic driving feature.",
"thought_1": "I have many types of sedans in my inventory, each with diverse features.",
@@ -294,11 +295,14 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
"action_1": {"name": "inventory", "input": "Yiem model A"},
"observation_1": "Yiem model A is in stock."
}
{"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
assistant
{
"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
It is also better to have simple searches corresponding to a single entity, making this the best action.",
"score": 10
}
user:
{
"question": "Do you have an all-in-one pen with 4 colors and a pencil for sale?",
"thought_1": "Let me check our inventory first to see if I have it.",
@@ -308,28 +312,44 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
"action_2": {"name": "chatbox", "input": "Yes, we do have a Pilot Dr. grip 4-in-1 pen and a Rotting pencil"},
"observation_1": "This is not what I wanted."
}
{"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
assistant:
{
"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
not a pen and a pencil seperately. A better search term should have been a 4-colors pen with a pencil, all-in-one.",
"score": 0
}
Let's begin!:
$(JSON3.write(state[:thoughtHistory]))
{"evaluation"
Let's begin!
"""
usermsg =
"""
$(JSON3.write(state[:thoughtHistory]))
"""
chathistory =
[
Dict(:name=> "system", :text=> systemmsg),
Dict(:name=> "user", :text=> usermsg)
]
# put in model format
prompt = formatLLMtext(chathistory, "llama3instruct")
prompt *=
"""
<|start_header_id|>assistant<|end_header_id|>
{
"""
pprint(prompt)
externalService = a.config[:externalservice][:text2textinstruct]
# apply LLM specific instruct format
externalService = a.config[:externalservice][:text2textinstruct]
llminfo = externalService[:llminfo]
prompt =
if llminfo[:name] == "llama3instruct"
formatLLMtext_llama3instruct("system", _prompt)
else
error("llm model name is not defied yet $(@__LINE__)")
end
msgMeta = GeneralUtils.generate_msgMeta(
a.config[:externalservice][:text2textinstruct][:mqtttopic],
externalService[:mqtttopic],
senderName= "evaluator",
senderId= a.id,
receiverName= "text2textinstruct",
@@ -377,6 +397,123 @@ function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T
end
error("evaluator failed to generate an evaluation")
end
# function evaluator(a::T1, state::T2)::Tuple{String, Integer} where {T1<:agent, T2<:AbstractDict}
# _prompt =
# """
# Analyze the trajectories of a solution to a question answering task. The trajectories are
# labeled by environmental observations about the situation, thoughts that can reason about
# the current situation and actions that can be three types:
# 1) winestock[query], which you can use to find wine in your inventory.
# 2) chatbox[text], which you can use to interact with the user.
# 3) recommendbox[answer], which returns your wine recommendation to the user.
# Given a question and a trajectory, evaluate its correctness and provide your reasoning and
# analysis in detail. Focus on the latest thought, action, and observation. Incomplete trajectories
# can be correct if the thoughts and actions so far are correct, even if the answer is not found
# yet. Do not generate additional thoughts or actions. Then ending with the correctness score s
# where s is an integer from 0 to 10.
# You should only respond in JSON format as describe below:
# {"evaluation": "your evaluation", "score": "your evaluation score"}
# Here are some examples:
# {
# "question": "I'm looking for a sedan with an automatic driving feature.",
# "thought_1": "I have many types of sedans in my inventory, each with diverse features.",
# "thought_2": "But there is only 1 model that has the feature customer wanted.",
# "thought_3": "I should check our inventory first to see if we have it.",
# "action_1": {"name": "inventory", "input": "Yiem model A"},
# "observation_1": "Yiem model A is in stock."
# }
# {"evaluation": "This trajectory is correct as it is reasonable to check an inventory for info provided in the question.
# It is also better to have simple searches corresponding to a single entity, making this the best action.",
# "score": 10
# }
# {
# "question": "Do you have an all-in-one pen with 4 colors and a pencil for sale?",
# "thought_1": "Let me check our inventory first to see if I have it.",
# "action_1": {"name": "inventory", "input": "pen with 4 color and a pencil."},
# "observation_1": "I found {1: "Pilot Dr. grip 4-in-1 pen", 2: "Rotting pencil"}",
# "thought_2": "Ok, I have what the user is asking. Let's tell the user.",
# "action_2": {"name": "chatbox", "input": "Yes, we do have a Pilot Dr. grip 4-in-1 pen and a Rotting pencil"},
# "observation_1": "This is not what I wanted."
# }
# {"evaluation": "This trajectory is incorrect as my search term should be related to a 4-colors pen with a pencil in it,
# not a pen and a pencil seperately. A better search term should have been a 4-colors pen with a pencil, all-in-one.",
# "score": 0
# }
# Let's begin!:
# $(JSON3.write(state[:thoughtHistory]))
# {"evaluation"
# """
# # apply LLM specific instruct format
# externalService = a.config[:externalservice][:text2textinstruct]
# llminfo = externalService[:llminfo]
# prompt =
# if llminfo[:name] == "llama3instruct"
# formatLLMtext_llama3instruct("system", _prompt)
# else
# error("llm model name is not defied yet $(@__LINE__)")
# end
# msgMeta = GeneralUtils.generate_msgMeta(
# a.config[:externalservice][:text2textinstruct][:mqtttopic],
# senderName= "evaluator",
# senderId= a.id,
# receiverName= "text2textinstruct",
# mqttBroker= a.config[:mqttServerInfo][:broker],
# mqttBrokerPort= a.config[:mqttServerInfo][:port],
# )
# outgoingMsg = Dict(
# :msgMeta=> msgMeta,
# :payload=> Dict(
# :text=> prompt,
# :kwargs=> Dict(
# :max_tokens=> 512,
# :stop=> ["<|eot_id|>"],
# )
# )
# )
# for attempt in 1:5
# try
# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg)
# _responseJsonStr = response[:response][:text]
# expectedJsonExample =
# """
# Here is an expected JSON format:
# {"evaluation": "...", "score": "..."}
# """
# responseJsonStr = jsoncorrection(a, _responseJsonStr, expectedJsonExample)
# evaluationDict = copy(JSON3.read(responseJsonStr))
# # check if dict has all required value
# dummya::AbstractString = evaluationDict[:evaluation]
# dummyb::Integer = evaluationDict[:score]
# return (evaluationDict[:evaluation], evaluationDict[:score])
# catch e
# io = IOBuffer()
# showerror(io, e)
# errorMsg = String(take!(io))
# st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
# println("")
# @warn "Attempt $attempt. Error occurred: $errorMsg\n$st"
# println("")
# end
# end
# error("evaluator failed to generate an evaluation")
# end
# """
@@ -784,7 +921,7 @@ function conversation(a::T, userinput::Dict) where {T<:agent}
while true
bestNextState, besttrajectory = runMCTS(a, a.plan[:currenttrajectory], decisionMaker,
evaluator, reflector, totalsample=2, maxDepth=2, maxiterations=2, explorationweight=1.0)
evaluator, reflector, totalsample=2, maxDepth=3, maxiterations=3, explorationweight=1.0)
a.plan[:activeplan] = bestNextState
latestActionKey, latestActionIndice =