update docstring

src/interface.jl

@@ -129,12 +129,13 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
 
   context = 
     """
-    <internal_context_for_LLM>
+    <internal_context_for_assistant>
       $(a.memory["scratchpad"])
-    </internal_context_for_LLM>
+    </internal_context_for_assistant>
     """
 
-  #WORKING add context to the latest message (in the front)
+  #WORKING add context to text of the latest message (in the front). 
+  # use for loop because in openai format, each msg may contain both text and image. 
   for d in enumerate(a.chathistory[end]["content"])
     if d["type"] == "text"
       d["text"] = context * d["text"]
@@ -150,16 +151,12 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
       println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
     end
 
-    unformatPrompt =
-      [
-        Dict("name" => "system", "text" => systemmsg),
-      ]
+    openai_msg = Dict(
+      "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
+      "messages" => a.chathistory,
+      "temperature" => 0.7
+    )
 
-    unformatPrompt = vcat(unformatPrompt, recentEvents)
-    # put in model format
-    prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName)
-    # add info
-    prompt = prompt * context
 
     response = a.context.text2textInstructLLM(prompt; senderId=a.id)
     response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)

src/type.jl

@@ -146,7 +146,7 @@ function sommelier(
   id::String= string(uuid4()),
   retailername::String= "retailer_name",
   maxHistoryMsg::Integer= 20,
-  chathistory::Vector{Dict{String, String}} = Vector{Dict{String, String}}(),
+  chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}(),
   llmFormatName::String= "granite3"
   )
 
@@ -164,17 +164,42 @@ function sommelier(
     )
 
   """ Memory
-    Ref: Chat prompt format https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/discussions/3
-    NO "system" message in chathistory because I want to add it at the inference time
-    chathistory= [
-      Dict("name"=>"user", "text"=> "Wassup!", "timestamp"=> Dates.now()),
-      Dict("name"=>"assistant", "text"=> "Hi I'm your assistant.", "timestamp"=> Dates.now()),
-    ]
+
+    Chat history use openai format as follow:
+
+      image1_path = "test/large_image.png"                           ---
+      image1_bytes = read(image1_path)                                 | this part must be done
+      image1_base64_string = base64encode(image1_bytes)                | in frontend 
+      mime_type = "image/png"                                          | not in agent code
+      data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" ---
+
+      chathistory= [
+        Dict(
+          "role" => "system",
+          "content" => [
+            Dict("type" => "text", "text" => "You are a helpful assiatant"),
+          ]
+        ),
+        Dict(
+          "role" => "user",
+          "content" => [
+            Dict("type" => "text", "text" => "<internal_context_for_assistant> 
+                                              LLM context here...
+                                              </internal_context_for_assistant>
+                                              Do you know this wine? Just give me brief intro."
+            ),
+            Dict(
+              "type" => "image_url",
+              "image_url" => Dict("url" => data1_uri)
+            ),
+          ]
+        )
+      ]
   """
   memory = Dict{String, Any}(
             "shortmem"=> OrderedDict{String, Any}(
               "db_search_result"=> Any[],
-              "scratchpad"=> "", #PENDING should be a dict e.g. Dict("database_search_result"=>Dict("wines"=> "", "search_query"=> ""))
+              "scratchpad"=> "",
             ),
             "events"=> Vector{Dict{String, Any}}(),
             "state"=> Dict{String, Any}(