diff --git a/src/interface.jl b/src/interface.jl index 84db415..7d1297b 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -82,7 +82,7 @@ julia> config = Dict( julia> output_thoughtDict = Dict( :thought_1 => "The customer wants to buy a bottle of wine. This is a good start!", - :action_1 => Dict{Symbol, Any}( + :action_1 => Dict{String, Any}( :action=>"CHATBOX", :input=>"What occasion are you buying the wine for?" ), @@ -106,7 +106,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 # if isempty(lessonDict) # "" # else - # lessons = Dict{Symbol, Any}() + # lessons = Dict{String, Any}() # for (k, v) in lessonDict # lessons[k] = lessonDict[k][:lesson] # end @@ -588,57 +588,31 @@ end """ Chat with llm. -# Arguments - `a::agent` - an agent - -# Return - None +# Example userinput -# Example -```jldoctest -julia> using JSON, UUIDs, Dates, FileIO, MQTTClient, ChatAgent -julia> const mqttBroker = "mqtt.yiem.cc" -julia> mqttclient, connection = MakeConnection(mqttBroker, 1883) -julia> tools=Dict( # update input format - "askbox"=>Dict( - :description => "Useful for when you need to ask the user for more context. Do not ask the user their own question.", - :input => "Input is a text in JSON format.{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}", - :output => "" , - :func => nothing, - ), +image_path = "test/large_image.png" +image_bytes = read(image_path) +base64_string = base64encode(image_bytes) + +# 2. Match the MIME type according to your file extension (e.g., png, jpeg) +mime_type = "image/png" +data_uri = "data:$(mime_type);base64,$(base64_string)" + +# 3. Construct payload with the Data URI +message => Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => "Describe this image for me"), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data_uri) ) -julia> msgMeta = Dict( - :msgPurpose=> "updateStatus", - :from=> "agent", - :to=> "llmAI", - :requestresponse=> "request", - :sendto=> "", # destination topic - :replyTo=> "agent/api/v0.1.0/txt/response", # requester ask responseer to send reply to this topic - :repondToMsgId=> "", # responseer is responseing to this msg id - :taskstatus=> "", # "complete", "fail", "waiting" or other status - :timestamp=> Dates.now(), - :msgId=> "$(uuid4())", -) -julia> a = ChatAgent.agentReflex( - "Jene", - mqttclient, - msgMeta, - agentConfigTopic, # I need a function to send msg to config topic to get load balancer - role=:sommelier, - tools=tools - ) -julia> newAgent = ChatAgent.agentReact(agent) -julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?") -``` + ] + ) -# TODO - - [] update docstring - - [] add recap to initialState for earlier completed question - -# Signature +- TODO add recap to initialState for earlier completed question """ -function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing, +function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing, maximumMsg=50) # place holder @@ -646,7 +620,20 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing, result = nothing chatresponse = nothing - if userinput === nothing + userinput = GeneralUtils.dictify(userinput) + + user_text_input, text_position = + if userinput !== nothing + for (i, d) in enumerate(userinput["content"]) + if d["type"] == "text" + (d["text"], i) + end + end + else + (nothing, nothing) + end + + if user_text_input === nothing # thinking loop until AI wants to communicate with the user chatresponse = nothing while chatresponse === nothing @@ -658,24 +645,24 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing, addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg) return chatresponse - elseif userinput[:text] == "newtopic" + elseif user_text_input == "newtopic" clearhistory(a) return "Okay. What shall we talk about?" else - userinput[:text] = GeneralUtils.remove_french_accents(userinput[:text]) - # add usermsg to a.chathistory - addNewMessage(a, "user", userinput[:text]; maximumMsg=maximumMsg) + userinput["content"][text_position] = GeneralUtils.remove_french_accents(user_text_input) + # add usermsg to a.chathistory but how do I handle images? + addNewMessage(a, "user", userinput; maximumMsg=maximumMsg) # add user activity to events memory - push!(a.memory[:events], - eventdict(; - event_description="the user talks to the assistant.", - timestamp=Dates.now(), - subject="user", - actionname="CHATBOX", - actioninput=userinput[:text], - ) - ) + # push!(a.memory[:events], + # eventdict(; + # event_description="the user talks to the assistant.", + # timestamp=Dates.now(), + # subject="user", + # actionname="CHATBOX", + # actioninput=userinput[:text], + # ) + # ) # thinking loop until AI wants to communicate with the user chatresponse = nothing @@ -742,11 +729,8 @@ end julia> ``` -# TODO - - [] update docstring - -# Signature -""" +TODO update docstring +""" # WORKING function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent} # a.memory[:recap] = generateSituationReport(a, a.context[:text2textInstructLLM]; skiprecent=0) thoughtDict = decisionMaker(a) diff --git a/src/llmfunction.jl b/src/llmfunction.jl index c028e02..13233d2 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -174,7 +174,7 @@ function virtualWineUserChatbox(config::T1, input::T2, virtualCustomerChatHistor pushfirst!(virtualCustomerChatHistory, Dict(:name=> "system", :text=> systemmsg)) # replace the :user key in chathistory to allow the virtual wine customer AI roleplay - chathistory::Vector{Dict{Symbol, Any}} = Vector{Dict{Symbol, Any}}() + chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}() for i in virtualCustomerChatHistory newdict = Dict() newdict[:name] = diff --git a/src/type.jl b/src/type.jl index 10260f6..7c4bfde 100644 --- a/src/type.jl +++ b/src/type.jl @@ -1,6 +1,6 @@ module type -export agent, sommelier, companion, virtualcustomer, appcontext +export agent, sommelier, companion, virtualcustomer, agentcontext using Dates, UUIDs, DataStructures, JSON, NATS using GeneralUtils @@ -8,11 +8,9 @@ using GeneralUtils # ---------------------------------------------- 100 --------------------------------------------- # -mutable struct appcontext - const connection::NATS.Connection - const text2textInstructLLMServiceSubject::String - getTextEmbedding::Function +mutable struct agentcontext text2textInstructLLM::Function + getTextEmbedding::Function executeSQL::Function similarSQLVectorDB::Function insertSQLVectorDB::Function @@ -28,14 +26,14 @@ mutable struct companion <: agent systemmsg::String # system message tools::Dict # tools maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized - chathistory::Vector{Dict{Symbol, Any}} - memory::Dict{Symbol, Any} + chathistory::Vector{Dict{String, Any}} + memory::Dict{String, Any} context::NamedTuple # NamedTuple of functions llmFormatName::String end function companion( - context::appcontext # NamedTuple of functions + context::agentcontext # NamedTuple of functions ; name::String= "Assistant", id::String= GeneralUtils.uuid4snakecase(), @@ -69,10 +67,10 @@ function companion( Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()), ] """ - memory = Dict{Symbol, Any}( - :events=> Vector{Dict{Symbol, Any}}(), - :state=> Dict{Symbol, Any}(), # state of the agent - :recap=> OrderedDict{Symbol, Any}(), # recap summary of the conversation + memory = Dict{String, Any}( + :events=> Vector{Dict{String, Any}}(), + :state=> Dict{String, Any}(), # state of the agent + :recap=> OrderedDict{String, Any}(), # recap summary of the conversation ) newAgent = companion( @@ -91,89 +89,58 @@ function companion( end - - -""" A sommelier agent. - -# Arguments - - `mqttClient::Client` - MQTTClient's client - - `msgMeta::Dict{Symbol, Any}` - A dict contain info about a message. - - `config::Dict{Symbol, Any}` - Config info for an agent. Contain mqtt topic for internal use and other info. - -# Keyword Arguments - - `name::String` - Agent's name - - `id::String` - Agent's ID - - `tools::Dict{Symbol, Any}` - Agent's tools - - `maxHistoryMsg::Integer` - max history message - -# Return - - `nothing` - -# Example -```jldoctest -julia> using YiemAgent, MQTTClient, GeneralUtils -julia> msgMeta = GeneralUtils.generate_msgMeta( - "N/A", - replyTopic = "/testtopic/prompt" - ) -julia> tools= Dict( - :chatbox=>Dict( - :name => "chatbox", - :description => "Useful only for when you need to ask the user for more info or context. Do not ask the user their own question.", - :input => "Input should be a text.", - :output => "" , - :func => nothing, - ), - ) -julia> agentConfig = Dict( - :receiveprompt=>Dict( - :mqtttopic=> "/testtopic/prompt", # topic to receive prompt i.e. frontend send msg to this topic - ), - :receiveinternal=>Dict( - :mqtttopic=> "/testtopic/internal", # receive topic for model's internal - ), - :text2text=>Dict( - :mqtttopic=> "/text2text/receive", - ), - ) -julia> client, connection = MakeConnection("test.mosquitto.org", 1883) -julia> agent = YiemAgent.bsommelier( - client, - msgMeta, - agentConfig, - name= "assistant", - id= "555", # agent instance id - tools=tools, - ) -``` - -# TODO -- [] update docstring -- [x] implement the function - -# Signature -""" mutable struct sommelier <: agent name::String # agent name id::String # agent id retailername::String tools::Dict maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized - chathistory::Vector{Dict{Symbol, Any}} - memory::Dict{Symbol, Any} - context # NamedTuple of functions + chathistory::Vector{Dict{String, Any}} + memory::Dict{String, Any} + context::agentcontext llmFormatName::String end + +""" A sommelier agent. +# Arguments + - `context::agentcontext` + Application context containing shared functions for LLM, SQL, and vector database operations. + +# Keyword Arguments + - `name::String` + Agent's name. Default: `"Assistant"` + - `id::String` + Agent's ID. Default: generated UUID string. + - `retailername::String` + Retailer name associated with the sommelier. Default: `"retailer_name"` + - `maxHistoryMsg::Integer` + Maximum history messages. Default: `20` + - `chathistory::Vector{Dict{Symbol, String}}` + Chat history. Default: empty vector. + - `llmFormatName::String` + LLM format name. Default: `"granite3"` + +# Return + - `sommelier`: An instantiated sommelier agent. + +# Example +```julia +julia> using YiemAgent +julia> context = agentcontext( + text2textInstructLLM, + getTextEmbedding, + executeSQL, + similarSQLVectorDB, + insertSQLVectorDB, + similarSommelierDecision, + insertSommelierDecision + ) +julia> agent = sommelier(context, name="WineExpert", id="123", retailername="MyWineShop") +``` +""" function sommelier( - context::appcontext, # app context + context::agentcontext, # app context ; name::String= "Assistant", id::String= string(uuid4()), @@ -204,15 +171,15 @@ function sommelier( Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()), ] """ - memory = Dict{Symbol, Any}( - :shortmem=> OrderedDict{Symbol, Any}( + memory = Dict{String, Any}( + :shortmem=> OrderedDict{String, Any}( :db_search_result=> Any[], :scratchpad=> "", #[PENDING] should be a dict e.g. Dict(:database_search_result=>Dict(:wines=> "", :search_query=> "")) ), - :events=> Vector{Dict{Symbol, Any}}(), - :state=> Dict{Symbol, Any}( + :events=> Vector{Dict{String, Any}}(), + :state=> Dict{String, Any}( ), - :recap=> OrderedDict{Symbol, Any}(), + :recap=> OrderedDict{String, Any}(), ) @@ -238,8 +205,8 @@ mutable struct virtualcustomer <: agent systemmsg::String # system message tools::Dict maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized - chathistory::Vector{Dict{Symbol, Any}} - memory::Dict{Symbol, Any} + chathistory::Vector{Dict{String, Any}} + memory::Dict{String, Any} context # NamedTuple of functions llmFormatName::String end @@ -281,13 +248,13 @@ function virtualcustomer( Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()), ] """ - memory = Dict{Symbol, Any}( - :shortmem=> OrderedDict{Symbol, Any}( + memory = Dict{String, Any}( + :shortmem=> OrderedDict{String, Any}( ), - :events=> Vector{Dict{Symbol, Any}}(), - :state=> Dict{Symbol, Any}( + :events=> Vector{Dict{String, Any}}(), + :state=> Dict{String, Any}( ), - :recap=> OrderedDict{Symbol, Any}(), + :recap=> OrderedDict{String, Any}(), ) newAgent = virtualcustomer( diff --git a/src/util.jl b/src/util.jl index 1ef6039..689809f 100644 --- a/src/util.jl +++ b/src/util.jl @@ -60,6 +60,17 @@ end """ Add new message to agent. + messages => Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => "Describe this image for me"), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data_uri) + ) + ] + ) + Arguments\n ----- a::agent @@ -76,44 +87,24 @@ end Example\n ----- ```jldoctest - julia> using YiemAgent, MQTTClient, GeneralUtils - julia> client, connection = MakeConnection("test.mosquitto.org", 1883) - julia> connect(client, connection) - julia> msgMeta = GeneralUtils.generate_msgMeta("testtopic") - julia> agentConfig = Dict( - :receiveprompt=>Dict( - :mqtttopic=> "testtopic/receive", - ), - :receiveinternal=>Dict( - :mqtttopic=> "testtopic/internal", - ), - :text2text=>Dict( - :mqtttopic=> "testtopic/text2text", - ), - ) - julia> a = YiemAgent.sommelier( - client, - msgMeta, - agentConfig, - ) - julia> YiemAgent.addNewMessage(a, "user", "hello") + ``` Signature\n ----- """ -function addNewMessage(a::T1, name::String, text::T2; - maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractString} +function addNewMessage(a::T1, name::String, userinput::T2; + maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractDict} if name ∉ ["system", "user", "assistant"] # guard against typo error("name is not in agent.availableRole $(@__LINE__)") end - #[PENDING] summarize the oldest 10 message + #TODO summarize the oldest 10 message if length(a.chathistory) > maximumMsg summarize(a.chathistory) else - d = Dict(:name=> name, :text=> text, :timestamp=> Dates.now()) - push!(a.chathistory, d) + userinput["timestamp"] = Dates.now() + push!(a.chathistory, userinput) end end @@ -241,7 +232,7 @@ function eventdict(; note::Union{String, Nothing}=nothing, ) - d = Dict{Symbol, Any}( + d = Dict{String, Any}( :event_description=> event_description, :timestamp=> timestamp, :subject=> subject, diff --git a/test/prompttest_1.jl b/test/prompttest_1.jl index c736b5b..229fbc6 100644 --- a/test/prompttest_1.jl +++ b/test/prompttest_1.jl @@ -66,7 +66,7 @@ tools=Dict( # update input format input = -OrderedDict{Symbol, Any}(:question => "Hello, I would like a get a bottle of wine", :thought_1 => "It's great that the user is looking for a bottle of wine. To give them a personalized recommendation, I need to know more about their preferences.", :action_1 => Dict{Symbol, Any}(:name => "chatbox", :input => "What occasion are you planning to use this wine for?"), :observation_1 => "We are holding a wedding party", :thought_2 => "A wedding party is a great occasion for a special bottle of wine. I need to know what type of food will be served, and how much the user is willing to spend.", :action_2 => Dict{Symbol, Any}(:name => "chatbox", :input => "What type of food will you be serving at the wedding?"), :observation_2 => "It will be Thai dishes.", :thought_3 => "The type of wine that pairs well with Thai dishes is usually a crisp and refreshing white wine, but I also need to consider the budget and personal preferences.", :action_3 => Dict{Symbol, Any}(:name => "chatbox", :input => "How much are you willing to spend on this bottle of wine?"), :observation_3 => "I would spend up to 50 bucks.", :thought_4 => "I have a good idea of the occasion, food, and budget. Now I need to know what type of wine the user is looking for.", :action_4 => Dict{Symbol, Any}(:name => "chatbox", :input => "What type of wine are you usually looking for? Red, White, Sparkling, Rose, Dessert or Fortified?"), :observation_4 => "I like full-bodied Red wine with low tannin.", :thought_5 => "Now that I have all the necessary information, I can start searching for a suitable wine in our inventory.", :action_5 => Dict{Symbol, Any}(:name => "winestock", :input => "red wine with low tannins"), :observation_5 => "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n", :thought_6 => "Now that I have the information about the wine, it's time to make a recommendation.", :action_6 => Dict{Symbol, Any}(:name => "recommendbox", :input => "El Enemigo Cabernet Franc 2019"), :observation_6 => "I don't like the one you recommend. I want dry wine.") +OrderedDict{String, Any}(:question => "Hello, I would like a get a bottle of wine", :thought_1 => "It's great that the user is looking for a bottle of wine. To give them a personalized recommendation, I need to know more about their preferences.", :action_1 => Dict{String, Any}(:name => "chatbox", :input => "What occasion are you planning to use this wine for?"), :observation_1 => "We are holding a wedding party", :thought_2 => "A wedding party is a great occasion for a special bottle of wine. I need to know what type of food will be served, and how much the user is willing to spend.", :action_2 => Dict{String, Any}(:name => "chatbox", :input => "What type of food will you be serving at the wedding?"), :observation_2 => "It will be Thai dishes.", :thought_3 => "The type of wine that pairs well with Thai dishes is usually a crisp and refreshing white wine, but I also need to consider the budget and personal preferences.", :action_3 => Dict{String, Any}(:name => "chatbox", :input => "How much are you willing to spend on this bottle of wine?"), :observation_3 => "I would spend up to 50 bucks.", :thought_4 => "I have a good idea of the occasion, food, and budget. Now I need to know what type of wine the user is looking for.", :action_4 => Dict{String, Any}(:name => "chatbox", :input => "What type of wine are you usually looking for? Red, White, Sparkling, Rose, Dessert or Fortified?"), :observation_4 => "I like full-bodied Red wine with low tannin.", :thought_5 => "Now that I have all the necessary information, I can start searching for a suitable wine in our inventory.", :action_5 => Dict{String, Any}(:name => "winestock", :input => "red wine with low tannins"), :observation_5 => "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n", :thought_6 => "Now that I have the information about the wine, it's time to make a recommendation.", :action_6 => Dict{String, Any}(:name => "recommendbox", :input => "El Enemigo Cabernet Franc 2019"), :observation_6 => "I don't like the one you recommend. I want dry wine.") result = YiemAgent.jsoncorrection(a, input) diff --git a/test/runtests.jl b/test/runtests.jl index e69de29..8c2658b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -0,0 +1,108 @@ + +using JSON, Dates, UUIDs, PrettyPrinting, Base64, NATS, HTTP +using GeneralUtils, msghandler + +config = JSON.parsefile("./appconfig.json") + +agent_conn = NATS.connect(config["nats_server_info"]["url"]) + +function text2text_instruct_llm(openai_msg::AbstractDict) + payloads = [("msg", openai_msg, "dictionary")] # List of tuples + _, msg_envelope_json_str = msghandler.smartpack( + config["externalService"]["servicesloadbalancer"]["nats"], + payloads; + msg_purpose="text2text", + broker_url=config["nats_server_info"]["url"], + fileserver_url=config["externalService"]["fileserver"]["url"]) + + reply = NATS.request(agent_conn, + config["externalService"]["servicesloadbalancer"]["nats"], + msg_envelope_json_str, timeout=120) + + incoming_env_json_str = String(reply.payload) + incoming_env = msghandler.smartunpack(incoming_env_json_str) + _llm_response = incoming_env["payloads"][1][2] + llm_response = _llm_response["choices"][1]["message"]["content"] + return llm_response +end + + + + +# 1. Read local file and encode to base64 string +image1_path = "test/large_image.png" +image1_bytes = read(image1_path) +image1_base64_string = base64encode(image1_bytes) + +# 2. Match the MIME type according to your file extension (e.g., png, jpeg) +mime_type = "image/png" +data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" + +# 3. Construct payload with the Data URI +openai_msg = Dict( + "model" => "gemma-4-E4B-it-UD-Q4_K_XL", + "messages" => [ + Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => "Do you know this wine? Just give me brief intro."), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data1_uri) + ) + ] + ) + ], + "temperature" => 0.7 +) + +llm_response = text2text_instruct_llm(openai_msg) + + + +# 1. Read local file and encode to base64 string +image2_path = "test/small_image.png" +image2_bytes = read(image2_path) +image2_base64_string = base64encode(image2_bytes) + +# 2. Match the MIME type according to your file extension (e.g., png, jpeg) +mime_type = "image/png" +data2_uri = "data:$(mime_type);base64,$(image2_base64_string)" + + +openai_msg = Dict( + "model" => "gemma-4-E4B-it-UD-Q4_K_XL", + "messages" => [ + Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => "Do you know this wine? Just give me brief intro."), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data1_uri) + ) + ] + ), + Dict( + "role" => "assistant", + "content" => [ + Dict("type" => "text", "text" => "Yes, I do!\n\nThis is **Asolo Bella Principessa**, a high-quality Italian sparkling wine made from the Prosecco region.\n\n### 🥂 Brief Intro\n\n* **What it is:** A Prosecco Superiore D.O.C.G., meaning it meets strict quality standards for a premium sparkling wine.\n* **Style:** It is a **Sparkling White Wine** and is designated as **Extra Dry**. This means it is crisp, refreshing, and has a dry finish (not overly sweet).\n* **Flavor Profile:** Expect bright, lively bubbles, often with notes of green apple, pear, and citrus.\n* **Best For:** It's a versatile wine, perfect for celebratory toasts, enjoying with appetizers (like seafood or charcuterie), or simply as a refreshing aperitivo."), + ] + ), + Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => "How does this wine differ from earlier wine?"), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data2_uri) + ) + ] + ), + ], + "temperature" => 0.7 +) + +llm_response = text2text_instruct_llm(openai_msg) + +