diff --git a/src/interface.jl b/src/interface.jl
index f06265a..9a15e07 100644
--- a/src/interface.jl
+++ b/src/interface.jl
@@ -129,12 +129,13 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
context =
"""
-
+
$(a.memory["scratchpad"])
-
+
"""
- #WORKING add context to the latest message (in the front)
+ #WORKING add context to text of the latest message (in the front).
+ # use for loop because in openai format, each msg may contain both text and image.
for d in enumerate(a.chathistory[end]["content"])
if d["type"] == "text"
d["text"] = context * d["text"]
@@ -149,17 +150,13 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
if attempt > 1
println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
-
- unformatPrompt =
- [
- Dict("name" => "system", "text" => systemmsg),
- ]
- unformatPrompt = vcat(unformatPrompt, recentEvents)
- # put in model format
- prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName)
- # add info
- prompt = prompt * context
+ openai_msg = Dict(
+ "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
+ "messages" => a.chathistory,
+ "temperature" => 0.7
+ )
+
response = a.context.text2textInstructLLM(prompt; senderId=a.id)
response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)
diff --git a/src/type.jl b/src/type.jl
index 57b48a1..76f4de3 100644
--- a/src/type.jl
+++ b/src/type.jl
@@ -146,7 +146,7 @@ function sommelier(
id::String= string(uuid4()),
retailername::String= "retailer_name",
maxHistoryMsg::Integer= 20,
- chathistory::Vector{Dict{String, String}} = Vector{Dict{String, String}}(),
+ chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}(),
llmFormatName::String= "granite3"
)
@@ -164,17 +164,42 @@ function sommelier(
)
""" Memory
- Ref: Chat prompt format https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/discussions/3
- NO "system" message in chathistory because I want to add it at the inference time
- chathistory= [
- Dict("name"=>"user", "text"=> "Wassup!", "timestamp"=> Dates.now()),
- Dict("name"=>"assistant", "text"=> "Hi I'm your assistant.", "timestamp"=> Dates.now()),
- ]
+
+ Chat history use openai format as follow:
+
+ image1_path = "test/large_image.png" ---
+ image1_bytes = read(image1_path) | this part must be done
+ image1_base64_string = base64encode(image1_bytes) | in frontend
+ mime_type = "image/png" | not in agent code
+ data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" ---
+
+ chathistory= [
+ Dict(
+ "role" => "system",
+ "content" => [
+ Dict("type" => "text", "text" => "You are a helpful assiatant"),
+ ]
+ ),
+ Dict(
+ "role" => "user",
+ "content" => [
+ Dict("type" => "text", "text" => "
+ LLM context here...
+
+ Do you know this wine? Just give me brief intro."
+ ),
+ Dict(
+ "type" => "image_url",
+ "image_url" => Dict("url" => data1_uri)
+ ),
+ ]
+ )
+ ]
"""
memory = Dict{String, Any}(
"shortmem"=> OrderedDict{String, Any}(
"db_search_result"=> Any[],
- "scratchpad"=> "", #PENDING should be a dict e.g. Dict("database_search_result"=>Dict("wines"=> "", "search_query"=> ""))
+ "scratchpad"=> "",
),
"events"=> Vector{Dict{String, Any}}(),
"state"=> Dict{String, Any}(