ton/YiemAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-06-24 08:47:36 +07:00
parent f6eaf4751b
commit 906afc6422
6 changed files with 244 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
src/interface.jl
View File

@@ -82,7 +82,7 @@ julia> config = Dict(
julia> output_thoughtDict = Dict(
:thought_1 => "The customer wants to buy a bottle of wine. This is a good start!",
:action_1 => Dict{Symbol, Any}(
:action_1 => Dict{String, Any}(
:action=>"CHATBOX",
:input=>"What occasion are you buying the wine for?"
),
@@ -106,7 +106,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
# if isempty(lessonDict)
# ""
# else
# lessons = Dict{Symbol, Any}()
# lessons = Dict{String, Any}()
# for (k, v) in lessonDict
# lessons[k] = lessonDict[k][:lesson]
# end
@@ -588,57 +588,31 @@ end
""" Chat with llm.
# Arguments
`a::agent`
an agent
# Return
None
# Example userinput
# Example
```jldoctest
julia> using JSON, UUIDs, Dates, FileIO, MQTTClient, ChatAgent
julia> const mqttBroker = "mqtt.yiem.cc"
julia> mqttclient, connection = MakeConnection(mqttBroker, 1883)
julia> tools=Dict( # update input format
"askbox"=>Dict(
:description => "<askbox tool description>Useful for when you need to ask the user for more context. Do not ask the user their own question.</askbox tool description>",
:input => "<input>Input is a text in JSON format.</input><input example>{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}</input example>",
:output => "" ,
:func => nothing,
),
image_path = "test/large_image.png"
image_bytes = read(image_path)
base64_string = base64encode(image_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png"
data_uri = "data:$(mime_type);base64,$(base64_string)"
# 3. Construct payload with the Data URI
message => Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Describe this image for me"),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data_uri)
)
julia> msgMeta = Dict(
:msgPurpose=> "updateStatus",
:from=> "agent",
:to=> "llmAI",
:requestresponse=> "request",
:sendto=> "", # destination topic
:replyTo=> "agent/api/v0.1.0/txt/response", # requester ask responseer to send reply to this topic
:repondToMsgId=> "", # responseer is responseing to this msg id
:taskstatus=> "", # "complete", "fail", "waiting" or other status
:timestamp=> Dates.now(),
:msgId=> "$(uuid4())",
)
julia> a = ChatAgent.agentReflex(
"Jene",
mqttclient,
msgMeta,
agentConfigTopic, # I need a function to send msg to config topic to get load balancer
role=:sommelier,
tools=tools
)
julia> newAgent = ChatAgent.agentReact(agent)
julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?")
```
]
)
# TODO
- [] update docstring
- [] add recap to initialState for earlier completed question
# Signature
- TODO add recap to initialState for earlier completed question
"""
function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
maximumMsg=50)
# place holder
@@ -646,7 +620,20 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
result = nothing
chatresponse = nothing
if userinput === nothing
userinput = GeneralUtils.dictify(userinput)
user_text_input, text_position =
if userinput !== nothing
for (i, d) in enumerate(userinput["content"])
if d["type"] == "text"
(d["text"], i)
end
end
else
(nothing, nothing)
end
if user_text_input === nothing
# thinking loop until AI wants to communicate with the user
chatresponse = nothing
while chatresponse === nothing
@@ -658,24 +645,24 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg)
return chatresponse
elseif userinput[:text] == "newtopic"
elseif user_text_input == "newtopic"
clearhistory(a)
return "Okay. What shall we talk about?"
else
userinput[:text] = GeneralUtils.remove_french_accents(userinput[:text])
# add usermsg to a.chathistory
addNewMessage(a, "user", userinput[:text]; maximumMsg=maximumMsg)
userinput["content"][text_position] = GeneralUtils.remove_french_accents(user_text_input)
# add usermsg to a.chathistory but how do I handle images?
addNewMessage(a, "user", userinput; maximumMsg=maximumMsg)
# add user activity to events memory
push!(a.memory[:events],
eventdict(;
event_description="the user talks to the assistant.",
timestamp=Dates.now(),
subject="user",
actionname="CHATBOX",
actioninput=userinput[:text],
)
)
# push!(a.memory[:events],
# eventdict(;
# event_description="the user talks to the assistant.",
# timestamp=Dates.now(),
# subject="user",
# actionname="CHATBOX",
# actioninput=userinput[:text],
# )
# )
# thinking loop until AI wants to communicate with the user
chatresponse = nothing
@@ -742,11 +729,8 @@ end
julia>
```
# TODO
- [] update docstring
# Signature
"""
TODO update docstring
""" # WORKING
function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent}
# a.memory[:recap] = generateSituationReport(a, a.context[:text2textInstructLLM]; skiprecent=0)
thoughtDict = decisionMaker(a)

2
src/llmfunction.jl
View File

@@ -174,7 +174,7 @@ function virtualWineUserChatbox(config::T1, input::T2, virtualCustomerChatHistor
pushfirst!(virtualCustomerChatHistory, Dict(:name=> "system", :text=> systemmsg))
# replace the :user key in chathistory to allow the virtual wine customer AI roleplay
chathistory::Vector{Dict{Symbol, Any}} = Vector{Dict{Symbol, Any}}()
chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}()
for i in virtualCustomerChatHistory
newdict = Dict()
newdict[:name] =

161
src/type.jl
View File

@@ -1,6 +1,6 @@
module type
export agent, sommelier, companion, virtualcustomer, appcontext
export agent, sommelier, companion, virtualcustomer, agentcontext
using Dates, UUIDs, DataStructures, JSON, NATS
using GeneralUtils
@@ -8,11 +8,9 @@ using GeneralUtils
# ---------------------------------------------- 100 --------------------------------------------- #
mutable struct appcontext
const connection::NATS.Connection
const text2textInstructLLMServiceSubject::String
getTextEmbedding::Function
mutable struct agentcontext
text2textInstructLLM::Function
getTextEmbedding::Function
executeSQL::Function
similarSQLVectorDB::Function
insertSQLVectorDB::Function
@@ -28,14 +26,14 @@ mutable struct companion <: agent
systemmsg::String # system message
tools::Dict # tools
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{Symbol, Any}}
memory::Dict{Symbol, Any}
chathistory::Vector{Dict{String, Any}}
memory::Dict{String, Any}
context::NamedTuple # NamedTuple of functions
llmFormatName::String
end
function companion(
context::appcontext # NamedTuple of functions
context::agentcontext # NamedTuple of functions
;
name::String= "Assistant",
id::String= GeneralUtils.uuid4snakecase(),
@@ -69,10 +67,10 @@ function companion(
Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()),
]
"""
memory = Dict{Symbol, Any}(
:events=> Vector{Dict{Symbol, Any}}(),
:state=> Dict{Symbol, Any}(), # state of the agent
:recap=> OrderedDict{Symbol, Any}(), # recap summary of the conversation
memory = Dict{String, Any}(
:events=> Vector{Dict{String, Any}}(),
:state=> Dict{String, Any}(), # state of the agent
:recap=> OrderedDict{String, Any}(), # recap summary of the conversation
)
newAgent = companion(
@@ -91,89 +89,58 @@ function companion(
end
""" A sommelier agent.
# Arguments
- `mqttClient::Client`
MQTTClient's client
- `msgMeta::Dict{Symbol, Any}`
A dict contain info about a message.
- `config::Dict{Symbol, Any}`
Config info for an agent. Contain mqtt topic for internal use and other info.
# Keyword Arguments
- `name::String`
Agent's name
- `id::String`
Agent's ID
- `tools::Dict{Symbol, Any}`
Agent's tools
- `maxHistoryMsg::Integer`
max history message
# Return
- `nothing`
# Example
```jldoctest
julia> using YiemAgent, MQTTClient, GeneralUtils
julia> msgMeta = GeneralUtils.generate_msgMeta(
"N/A",
replyTopic = "/testtopic/prompt"
)
julia> tools= Dict(
:chatbox=>Dict(
:name => "chatbox",
:description => "Useful only for when you need to ask the user for more info or context. Do not ask the user their own question.",
:input => "Input should be a text.",
:output => "" ,
:func => nothing,
),
)
julia> agentConfig = Dict(
:receiveprompt=>Dict(
:mqtttopic=> "/testtopic/prompt", # topic to receive prompt i.e. frontend send msg to this topic
),
:receiveinternal=>Dict(
:mqtttopic=> "/testtopic/internal", # receive topic for model's internal
),
:text2text=>Dict(
:mqtttopic=> "/text2text/receive",
),
)
julia> client, connection = MakeConnection("test.mosquitto.org", 1883)
julia> agent = YiemAgent.bsommelier(
client,
msgMeta,
agentConfig,
name= "assistant",
id= "555", # agent instance id
tools=tools,
)
```
# TODO
- [] update docstring
- [x] implement the function
# Signature
"""
mutable struct sommelier <: agent
name::String # agent name
id::String # agent id
retailername::String
tools::Dict
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{Symbol, Any}}
memory::Dict{Symbol, Any}
context # NamedTuple of functions
chathistory::Vector{Dict{String, Any}}
memory::Dict{String, Any}
context::agentcontext
llmFormatName::String
end
""" A sommelier agent.
# Arguments
- `context::agentcontext`
Application context containing shared functions for LLM, SQL, and vector database operations.
# Keyword Arguments
- `name::String`
Agent's name. Default: `"Assistant"`
- `id::String`
Agent's ID. Default: generated UUID string.
- `retailername::String`
Retailer name associated with the sommelier. Default: `"retailer_name"`
- `maxHistoryMsg::Integer`
Maximum history messages. Default: `20`
- `chathistory::Vector{Dict{Symbol, String}}`
Chat history. Default: empty vector.
- `llmFormatName::String`
LLM format name. Default: `"granite3"`
# Return
- `sommelier`: An instantiated sommelier agent.
# Example
```julia
julia> using YiemAgent
julia> context = agentcontext(
text2textInstructLLM,
getTextEmbedding,
executeSQL,
similarSQLVectorDB,
insertSQLVectorDB,
similarSommelierDecision,
insertSommelierDecision
)
julia> agent = sommelier(context, name="WineExpert", id="123", retailername="MyWineShop")
```
"""
function sommelier(
context::appcontext, # app context
context::agentcontext, # app context
;
name::String= "Assistant",
id::String= string(uuid4()),
@@ -204,15 +171,15 @@ function sommelier(
Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()),
]
"""
memory = Dict{Symbol, Any}(
:shortmem=> OrderedDict{Symbol, Any}(
memory = Dict{String, Any}(
:shortmem=> OrderedDict{String, Any}(
:db_search_result=> Any[],
:scratchpad=> "", #[PENDING] should be a dict e.g. Dict(:database_search_result=>Dict(:wines=> "", :search_query=> ""))
),
:events=> Vector{Dict{Symbol, Any}}(),
:state=> Dict{Symbol, Any}(
:events=> Vector{Dict{String, Any}}(),
:state=> Dict{String, Any}(
),
:recap=> OrderedDict{Symbol, Any}(),
:recap=> OrderedDict{String, Any}(),
)
@@ -238,8 +205,8 @@ mutable struct virtualcustomer <: agent
systemmsg::String # system message
tools::Dict
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{Symbol, Any}}
memory::Dict{Symbol, Any}
chathistory::Vector{Dict{String, Any}}
memory::Dict{String, Any}
context # NamedTuple of functions
llmFormatName::String
end
@@ -281,13 +248,13 @@ function virtualcustomer(
Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()),
]
"""
memory = Dict{Symbol, Any}(
:shortmem=> OrderedDict{Symbol, Any}(
memory = Dict{String, Any}(
:shortmem=> OrderedDict{String, Any}(
),
:events=> Vector{Dict{Symbol, Any}}(),
:state=> Dict{Symbol, Any}(
:events=> Vector{Dict{String, Any}}(),
:state=> Dict{String, Any}(
),
:recap=> OrderedDict{Symbol, Any}(),
:recap=> OrderedDict{String, Any}(),
)
newAgent = virtualcustomer(

45
src/util.jl
View File

@@ -60,6 +60,17 @@ end
""" Add new message to agent.
messages => Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Describe this image for me"),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data_uri)
)
]
)
Arguments\n
-----
a::agent
@@ -76,44 +87,24 @@ end
Example\n
-----
```jldoctest
julia> using YiemAgent, MQTTClient, GeneralUtils
julia> client, connection = MakeConnection("test.mosquitto.org", 1883)
julia> connect(client, connection)
julia> msgMeta = GeneralUtils.generate_msgMeta("testtopic")
julia> agentConfig = Dict(
:receiveprompt=>Dict(
:mqtttopic=> "testtopic/receive",
),
:receiveinternal=>Dict(
:mqtttopic=> "testtopic/internal",
),
:text2text=>Dict(
:mqtttopic=> "testtopic/text2text",
),
)
julia> a = YiemAgent.sommelier(
client,
msgMeta,
agentConfig,
)
julia> YiemAgent.addNewMessage(a, "user", "hello")
```
Signature\n
-----
"""
function addNewMessage(a::T1, name::String, text::T2;
maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractString}
function addNewMessage(a::T1, name::String, userinput::T2;
maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractDict}
if name ∉ ["system", "user", "assistant"] # guard against typo
error("name is not in agent.availableRole $(@__LINE__)")
end
#[PENDING] summarize the oldest 10 message
#TODO summarize the oldest 10 message
if length(a.chathistory) > maximumMsg
summarize(a.chathistory)
else
d = Dict(:name=> name, :text=> text, :timestamp=> Dates.now())
push!(a.chathistory, d)
userinput["timestamp"] = Dates.now()
push!(a.chathistory, userinput)
end
end
@@ -241,7 +232,7 @@ function eventdict(;
note::Union{String, Nothing}=nothing,
)
d = Dict{Symbol, Any}(
d = Dict{String, Any}(
:event_description=> event_description,
:timestamp=> timestamp,
:subject=> subject,

2
test/prompttest_1.jl
View File

@@ -66,7 +66,7 @@ tools=Dict( # update input format
input =
OrderedDict{Symbol, Any}(:question => "Hello, I would like a get a bottle of wine", :thought_1 => "It's great that the user is looking for a bottle of wine. To give them a personalized recommendation, I need to know more about their preferences.", :action_1 => Dict{Symbol, Any}(:name => "chatbox", :input => "What occasion are you planning to use this wine for?"), :observation_1 => "We are holding a wedding party", :thought_2 => "A wedding party is a great occasion for a special bottle of wine. I need to know what type of food will be served, and how much the user is willing to spend.", :action_2 => Dict{Symbol, Any}(:name => "chatbox", :input => "What type of food will you be serving at the wedding?"), :observation_2 => "It will be Thai dishes.", :thought_3 => "The type of wine that pairs well with Thai dishes is usually a crisp and refreshing white wine, but I also need to consider the budget and personal preferences.", :action_3 => Dict{Symbol, Any}(:name => "chatbox", :input => "How much are you willing to spend on this bottle of wine?"), :observation_3 => "I would spend up to 50 bucks.", :thought_4 => "I have a good idea of the occasion, food, and budget. Now I need to know what type of wine the user is looking for.", :action_4 => Dict{Symbol, Any}(:name => "chatbox", :input => "What type of wine are you usually looking for? Red, White, Sparkling, Rose, Dessert or Fortified?"), :observation_4 => "I like full-bodied Red wine with low tannin.", :thought_5 => "Now that I have all the necessary information, I can start searching for a suitable wine in our inventory.", :action_5 => Dict{Symbol, Any}(:name => "winestock", :input => "red wine with low tannins"), :observation_5 => "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n", :thought_6 => "Now that I have the information about the wine, it's time to make a recommendation.", :action_6 => Dict{Symbol, Any}(:name => "recommendbox", :input => "El Enemigo Cabernet Franc 2019"), :observation_6 => "I don't like the one you recommend. I want dry wine.")
OrderedDict{String, Any}(:question => "Hello, I would like a get a bottle of wine", :thought_1 => "It's great that the user is looking for a bottle of wine. To give them a personalized recommendation, I need to know more about their preferences.", :action_1 => Dict{String, Any}(:name => "chatbox", :input => "What occasion are you planning to use this wine for?"), :observation_1 => "We are holding a wedding party", :thought_2 => "A wedding party is a great occasion for a special bottle of wine. I need to know what type of food will be served, and how much the user is willing to spend.", :action_2 => Dict{String, Any}(:name => "chatbox", :input => "What type of food will you be serving at the wedding?"), :observation_2 => "It will be Thai dishes.", :thought_3 => "The type of wine that pairs well with Thai dishes is usually a crisp and refreshing white wine, but I also need to consider the budget and personal preferences.", :action_3 => Dict{String, Any}(:name => "chatbox", :input => "How much are you willing to spend on this bottle of wine?"), :observation_3 => "I would spend up to 50 bucks.", :thought_4 => "I have a good idea of the occasion, food, and budget. Now I need to know what type of wine the user is looking for.", :action_4 => Dict{String, Any}(:name => "chatbox", :input => "What type of wine are you usually looking for? Red, White, Sparkling, Rose, Dessert or Fortified?"), :observation_4 => "I like full-bodied Red wine with low tannin.", :thought_5 => "Now that I have all the necessary information, I can start searching for a suitable wine in our inventory.", :action_5 => Dict{String, Any}(:name => "winestock", :input => "red wine with low tannins"), :observation_5 => "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n", :thought_6 => "Now that I have the information about the wine, it's time to make a recommendation.", :action_6 => Dict{String, Any}(:name => "recommendbox", :input => "El Enemigo Cabernet Franc 2019"), :observation_6 => "I don't like the one you recommend. I want dry wine.")
result = YiemAgent.jsoncorrection(a, input)

108
test/runtests.jl
View File

@@ -0,0 +1,108 @@
using JSON, Dates, UUIDs, PrettyPrinting, Base64, NATS, HTTP
using GeneralUtils, msghandler
config = JSON.parsefile("./appconfig.json")
agent_conn = NATS.connect(config["nats_server_info"]["url"])
function text2text_instruct_llm(openai_msg::AbstractDict)
payloads = [("msg", openai_msg, "dictionary")] # List of tuples
_, msg_envelope_json_str = msghandler.smartpack(
config["externalService"]["servicesloadbalancer"]["nats"],
payloads;
msg_purpose="text2text",
broker_url=config["nats_server_info"]["url"],
fileserver_url=config["externalService"]["fileserver"]["url"])
reply = NATS.request(agent_conn,
config["externalService"]["servicesloadbalancer"]["nats"],
msg_envelope_json_str, timeout=120)
incoming_env_json_str = String(reply.payload)
incoming_env = msghandler.smartunpack(incoming_env_json_str)
_llm_response = incoming_env["payloads"][1][2]
llm_response = _llm_response["choices"][1]["message"]["content"]
return llm_response
end
# 1. Read local file and encode to base64 string
image1_path = "test/large_image.png"
image1_bytes = read(image1_path)
image1_base64_string = base64encode(image1_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png"
data1_uri = "data:$(mime_type);base64,$(image1_base64_string)"
# 3. Construct payload with the Data URI
openai_msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => [
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Do you know this wine? Just give me brief intro."),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data1_uri)
)
]
)
],
"temperature" => 0.7
)
llm_response = text2text_instruct_llm(openai_msg)
# 1. Read local file and encode to base64 string
image2_path = "test/small_image.png"
image2_bytes = read(image2_path)
image2_base64_string = base64encode(image2_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png"
data2_uri = "data:$(mime_type);base64,$(image2_base64_string)"
openai_msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => [
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Do you know this wine? Just give me brief intro."),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data1_uri)
)
]
),
Dict(
"role" => "assistant",
"content" => [
Dict("type" => "text", "text" => "Yes, I do!\n\nThis is **Asolo Bella Principessa**, a high-quality Italian sparkling wine made from the Prosecco region.\n\n### 🥂 Brief Intro\n\n* **What it is:** A Prosecco Superiore D.O.C.G., meaning it meets strict quality standards for a premium sparkling wine.\n* **Style:** It is a **Sparkling White Wine** and is designated as **Extra Dry**. This means it is crisp, refreshing, and has a dry finish (not overly sweet).\n* **Flavor Profile:** Expect bright, lively bubbles, often with notes of green apple, pear, and citrus.\n* **Best For:** It's a versatile wine, perfect for celebratory toasts, enjoying with appetizers (like seafood or charcuterie), or simply as a refreshing aperitivo."),
]
),
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "How does this wine differ from earlier wine?"),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data2_uri)
)
]
),
],
"temperature" => 0.7
)
llm_response = text2text_instruct_llm(openai_msg)