update

src/interface.jl

@@ -82,7 +82,7 @@ julia>  config = Dict(
 
 julia> output_thoughtDict = Dict(
         :thought_1     => "The customer wants to buy a bottle of wine. This is a good start!",
-        :action_1      => Dict{Symbol, Any}(
+        :action_1      => Dict{String, Any}(
                             :action=>"CHATBOX", 
                             :input=>"What occasion are you buying the wine for?"
                             ),
@@ -106,7 +106,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
   # if isempty(lessonDict)
   #   ""
   # else
-  #   lessons = Dict{Symbol, Any}()
+  #   lessons = Dict{String, Any}()
   #   for (k, v) in lessonDict
   #     lessons[k] = lessonDict[k][:lesson]
   #   end
@@ -588,57 +588,31 @@ end
 
 """ Chat with llm.
 
-# Arguments
-  `a::agent`
-    an agent
- 
-# Return
-  None
+# Example userinput
 
-# Example
-```jldoctest
-julia> using JSON, UUIDs, Dates, FileIO, MQTTClient, ChatAgent
-julia> const mqttBroker = "mqtt.yiem.cc"
-julia> mqttclient, connection = MakeConnection(mqttBroker, 1883)
-julia> tools=Dict( # update input format
-        "askbox"=>Dict(
-          :description => "<askbox tool description>Useful for when you need to ask the user for more context. Do not ask the user their own question.</askbox tool description>",
-          :input => "<input>Input is a text in JSON format.</input><input example>{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}</input example>",
-          :output => "" ,
-          :func => nothing,
-        ),
+image_path = "test/large_image.png"
+image_bytes = read(image_path)
+base64_string = base64encode(image_bytes)
+
+# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
+mime_type = "image/png" 
+data_uri = "data:$(mime_type);base64,$(base64_string)"
+
+# 3. Construct payload with the Data URI
+message => Dict(
+    "role" => "user",
+    "content" => [
+      Dict("type" => "text", "text" => "Describe this image for me"),
+      Dict(
+        "type" => "image_url",
+        "image_url" => Dict("url" => data_uri)
       )
-julia> msgMeta = Dict(
-  :msgPurpose=> "updateStatus",
-  :from=> "agent",       
-  :to=> "llmAI",
-  :requestresponse=> "request",
-  :sendto=> "",      # destination topic
-  :replyTo=> "agent/api/v0.1.0/txt/response",     # requester ask responseer to send reply to this topic
-  :repondToMsgId=> "",   # responseer is responseing to this msg id
-  :taskstatus=> "",       # "complete", "fail", "waiting" or other status
-  :timestamp=> Dates.now(),
-  :msgId=> "$(uuid4())",
-)
-julia> a = ChatAgent.agentReflex(
-        "Jene",
-        mqttclient,
-        msgMeta,
-        agentConfigTopic, # I need a function to send msg to config topic to get load balancer
-        role=:sommelier,
-        tools=tools
-      )
-julia> newAgent = ChatAgent.agentReact(agent)
-julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?")
-```
+    ]
+  )
 
-# TODO
-  - [] update docstring
-  - [] add recap to initialState for earlier completed question
-
-# Signature
+- TODO add recap to initialState for earlier completed question
 """
-function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing, 
+function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
                       maximumMsg=50)
 
   # place holder
@@ -646,7 +620,20 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
   result = nothing
   chatresponse = nothing
 
-  if userinput === nothing
+  userinput = GeneralUtils.dictify(userinput)
+
+  user_text_input, text_position =
+    if userinput !== nothing
+      for (i, d) in enumerate(userinput["content"])
+        if d["type"] == "text" 
+          (d["text"], i)
+        end
+      end
+    else
+      (nothing, nothing)
+    end
+
+  if user_text_input === nothing
     # thinking loop until AI wants to communicate with the user
     chatresponse = nothing
     while chatresponse === nothing
@@ -658,24 +645,24 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
     addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg)
 
     return chatresponse
-  elseif userinput[:text] == "newtopic"
+  elseif user_text_input == "newtopic"
     clearhistory(a)
     return "Okay. What shall we talk about?"
   else
-    userinput[:text] = GeneralUtils.remove_french_accents(userinput[:text])
-    # add usermsg to a.chathistory
-    addNewMessage(a, "user", userinput[:text]; maximumMsg=maximumMsg)
+    userinput["content"][text_position] = GeneralUtils.remove_french_accents(user_text_input)
+    # add usermsg to a.chathistory but how do I handle images?
+    addNewMessage(a, "user", userinput; maximumMsg=maximumMsg)
 
     # add user activity to events memory
-    push!(a.memory[:events],
-      eventdict(;
-        event_description="the user talks to the assistant.",
-        timestamp=Dates.now(),
-        subject="user",
-        actionname="CHATBOX",
-        actioninput=userinput[:text],
-      )
-    )
+    # push!(a.memory[:events],
+    #   eventdict(;
+    #     event_description="the user talks to the assistant.",
+    #     timestamp=Dates.now(),
+    #     subject="user",
+    #     actionname="CHATBOX",
+    #     actioninput=userinput[:text],
+    #   )
+    # )
 
     # thinking loop until AI wants to communicate with the user
     chatresponse = nothing
@@ -742,11 +729,8 @@ end
 julia> 
 ```
 
-# TODO
-  - [] update docstring
-
-# Signature
-"""
+TODO update docstring
+""" # WORKING
 function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent}
   # a.memory[:recap] = generateSituationReport(a, a.context[:text2textInstructLLM]; skiprecent=0)
   thoughtDict = decisionMaker(a)