diff --git a/src/interface.jl b/src/interface.jl index f06265a..9a15e07 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -129,12 +129,13 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 context = """ - + $(a.memory["scratchpad"]) - + """ - #WORKING add context to the latest message (in the front) + #WORKING add context to text of the latest message (in the front). + # use for loop because in openai format, each msg may contain both text and image. for d in enumerate(a.chathistory[end]["content"]) if d["type"] == "text" d["text"] = context * d["text"] @@ -149,17 +150,13 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 if attempt > 1 println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - - unformatPrompt = - [ - Dict("name" => "system", "text" => systemmsg), - ] - unformatPrompt = vcat(unformatPrompt, recentEvents) - # put in model format - prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) - # add info - prompt = prompt * context + openai_msg = Dict( + "model" => "gemma-4-E4B-it-UD-Q4_K_XL", + "messages" => a.chathistory, + "temperature" => 0.7 + ) + response = a.context.text2textInstructLLM(prompt; senderId=a.id) response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) diff --git a/src/type.jl b/src/type.jl index 57b48a1..76f4de3 100644 --- a/src/type.jl +++ b/src/type.jl @@ -146,7 +146,7 @@ function sommelier( id::String= string(uuid4()), retailername::String= "retailer_name", maxHistoryMsg::Integer= 20, - chathistory::Vector{Dict{String, String}} = Vector{Dict{String, String}}(), + chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}(), llmFormatName::String= "granite3" ) @@ -164,17 +164,42 @@ function sommelier( ) """ Memory - Ref: Chat prompt format https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/discussions/3 - NO "system" message in chathistory because I want to add it at the inference time - chathistory= [ - Dict("name"=>"user", "text"=> "Wassup!", "timestamp"=> Dates.now()), - Dict("name"=>"assistant", "text"=> "Hi I'm your assistant.", "timestamp"=> Dates.now()), - ] + + Chat history use openai format as follow: + + image1_path = "test/large_image.png" --- + image1_bytes = read(image1_path) | this part must be done + image1_base64_string = base64encode(image1_bytes) | in frontend + mime_type = "image/png" | not in agent code + data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" --- + + chathistory= [ + Dict( + "role" => "system", + "content" => [ + Dict("type" => "text", "text" => "You are a helpful assiatant"), + ] + ), + Dict( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => " + LLM context here... + + Do you know this wine? Just give me brief intro." + ), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data1_uri) + ), + ] + ) + ] """ memory = Dict{String, Any}( "shortmem"=> OrderedDict{String, Any}( "db_search_result"=> Any[], - "scratchpad"=> "", #PENDING should be a dict e.g. Dict("database_search_result"=>Dict("wines"=> "", "search_query"=> "")) + "scratchpad"=> "", ), "events"=> Vector{Dict{String, Any}}(), "state"=> Dict{String, Any}(