ton/YiemAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-06-24 08:47:36 +07:00
parent f6eaf4751b
commit 906afc6422
6 changed files with 244 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
src/interface.jl
View File

@@ -82,7 +82,7 @@ julia> config = Dict(
julia> output_thoughtDict = Dict( julia> output_thoughtDict = Dict(
:thought_1 => "The customer wants to buy a bottle of wine. This is a good start!", :thought_1 => "The customer wants to buy a bottle of wine. This is a good start!",
:action_1 => Dict{Symbol, Any}( :action_1 => Dict{String, Any}(
:action=>"CHATBOX", :action=>"CHATBOX",
:input=>"What occasion are you buying the wine for?" :input=>"What occasion are you buying the wine for?"
), ),
@@ -106,7 +106,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
# if isempty(lessonDict) # if isempty(lessonDict)
# "" # ""
# else # else
# lessons = Dict{Symbol, Any}() # lessons = Dict{String, Any}()
# for (k, v) in lessonDict # for (k, v) in lessonDict
# lessons[k] = lessonDict[k][:lesson] # lessons[k] = lessonDict[k][:lesson]
# end # end
@@ -588,55 +588,29 @@ end
""" Chat with llm. """ Chat with llm.
# Arguments # Example userinput
`a::agent`
an agent
# Return image_path = "test/large_image.png"
None image_bytes = read(image_path)
base64_string = base64encode(image_bytes)
# Example # 2. Match the MIME type according to your file extension (e.g., png, jpeg)
```jldoctest mime_type = "image/png"
julia> using JSON, UUIDs, Dates, FileIO, MQTTClient, ChatAgent data_uri = "data:$(mime_type);base64,$(base64_string)"
julia> const mqttBroker = "mqtt.yiem.cc"
julia> mqttclient, connection = MakeConnection(mqttBroker, 1883) # 3. Construct payload with the Data URI
julia> tools=Dict( # update input format message => Dict(
"askbox"=>Dict( "role" => "user",
:description => "<askbox tool description>Useful for when you need to ask the user for more context. Do not ask the user their own question.</askbox tool description>", "content" => [
:input => "<input>Input is a text in JSON format.</input><input example>{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}</input example>", Dict("type" => "text", "text" => "Describe this image for me"),
:output => "" , Dict(
:func => nothing, "type" => "image_url",
), "image_url" => Dict("url" => data_uri)
) )
julia> msgMeta = Dict( ]
:msgPurpose=> "updateStatus", )
:from=> "agent",
:to=> "llmAI",
:requestresponse=> "request",
:sendto=> "", # destination topic
:replyTo=> "agent/api/v0.1.0/txt/response", # requester ask responseer to send reply to this topic
:repondToMsgId=> "", # responseer is responseing to this msg id
:taskstatus=> "", # "complete", "fail", "waiting" or other status
:timestamp=> Dates.now(),
:msgId=> "$(uuid4())",
)
julia> a = ChatAgent.agentReflex(
"Jene",
mqttclient,
msgMeta,
agentConfigTopic, # I need a function to send msg to config topic to get load balancer
role=:sommelier,
tools=tools
)
julia> newAgent = ChatAgent.agentReact(agent)
julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?")
```
# TODO - TODO add recap to initialState for earlier completed question
- [] update docstring
- [] add recap to initialState for earlier completed question
# Signature
""" """
function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing, function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
maximumMsg=50) maximumMsg=50)
@@ -646,7 +620,20 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
result = nothing result = nothing
chatresponse = nothing chatresponse = nothing
if userinput === nothing userinput = GeneralUtils.dictify(userinput)
user_text_input, text_position =
if userinput !== nothing
for (i, d) in enumerate(userinput["content"])
if d["type"] == "text"
(d["text"], i)
end
end
else
(nothing, nothing)
end
if user_text_input === nothing
# thinking loop until AI wants to communicate with the user # thinking loop until AI wants to communicate with the user
chatresponse = nothing chatresponse = nothing
while chatresponse === nothing while chatresponse === nothing
@@ -658,24 +645,24 @@ function conversation(a::sommelier; userinput::Union{Dict, Nothing}=nothing,
addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg) addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg)
return chatresponse return chatresponse
elseif userinput[:text] == "newtopic" elseif user_text_input == "newtopic"
clearhistory(a) clearhistory(a)
return "Okay. What shall we talk about?" return "Okay. What shall we talk about?"
else else
userinput[:text] = GeneralUtils.remove_french_accents(userinput[:text]) userinput["content"][text_position] = GeneralUtils.remove_french_accents(user_text_input)
# add usermsg to a.chathistory # add usermsg to a.chathistory but how do I handle images?
addNewMessage(a, "user", userinput[:text]; maximumMsg=maximumMsg) addNewMessage(a, "user", userinput; maximumMsg=maximumMsg)
# add user activity to events memory # add user activity to events memory
push!(a.memory[:events], # push!(a.memory[:events],
eventdict(; # eventdict(;
event_description="the user talks to the assistant.", # event_description="the user talks to the assistant.",
timestamp=Dates.now(), # timestamp=Dates.now(),
subject="user", # subject="user",
actionname="CHATBOX", # actionname="CHATBOX",
actioninput=userinput[:text], # actioninput=userinput[:text],
) # )
) # )
# thinking loop until AI wants to communicate with the user # thinking loop until AI wants to communicate with the user
chatresponse = nothing chatresponse = nothing
@@ -742,11 +729,8 @@ end
julia> julia>
``` ```
# TODO TODO update docstring
- [] update docstring """ # WORKING
# Signature
"""
function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent} function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent}
# a.memory[:recap] = generateSituationReport(a, a.context[:text2textInstructLLM]; skiprecent=0) # a.memory[:recap] = generateSituationReport(a, a.context[:text2textInstructLLM]; skiprecent=0)
thoughtDict = decisionMaker(a) thoughtDict = decisionMaker(a)

2
src/llmfunction.jl
View File

@@ -174,7 +174,7 @@ function virtualWineUserChatbox(config::T1, input::T2, virtualCustomerChatHistor
pushfirst!(virtualCustomerChatHistory, Dict(:name=> "system", :text=> systemmsg)) pushfirst!(virtualCustomerChatHistory, Dict(:name=> "system", :text=> systemmsg))
# replace the :user key in chathistory to allow the virtual wine customer AI roleplay # replace the :user key in chathistory to allow the virtual wine customer AI roleplay
chathistory::Vector{Dict{Symbol, Any}} = Vector{Dict{Symbol, Any}}() chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}()
for i in virtualCustomerChatHistory for i in virtualCustomerChatHistory
newdict = Dict() newdict = Dict()
newdict[:name] = newdict[:name] =

161
src/type.jl
View File

@@ -1,6 +1,6 @@
module type module type
export agent, sommelier, companion, virtualcustomer, appcontext export agent, sommelier, companion, virtualcustomer, agentcontext
using Dates, UUIDs, DataStructures, JSON, NATS using Dates, UUIDs, DataStructures, JSON, NATS
using GeneralUtils using GeneralUtils
@@ -8,11 +8,9 @@ using GeneralUtils
# ---------------------------------------------- 100 --------------------------------------------- # # ---------------------------------------------- 100 --------------------------------------------- #
mutable struct appcontext mutable struct agentcontext
const connection::NATS.Connection
const text2textInstructLLMServiceSubject::String
getTextEmbedding::Function
text2textInstructLLM::Function text2textInstructLLM::Function
getTextEmbedding::Function
executeSQL::Function executeSQL::Function
similarSQLVectorDB::Function similarSQLVectorDB::Function
insertSQLVectorDB::Function insertSQLVectorDB::Function
@@ -28,14 +26,14 @@ mutable struct companion <: agent
systemmsg::String # system message systemmsg::String # system message
tools::Dict # tools tools::Dict # tools
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{Symbol, Any}} chathistory::Vector{Dict{String, Any}}
memory::Dict{Symbol, Any} memory::Dict{String, Any}
context::NamedTuple # NamedTuple of functions context::NamedTuple # NamedTuple of functions
llmFormatName::String llmFormatName::String
end end
function companion( function companion(
context::appcontext # NamedTuple of functions context::agentcontext # NamedTuple of functions
; ;
name::String= "Assistant", name::String= "Assistant",
id::String= GeneralUtils.uuid4snakecase(), id::String= GeneralUtils.uuid4snakecase(),
@@ -69,10 +67,10 @@ function companion(
Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()), Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()),
] ]
""" """
memory = Dict{Symbol, Any}( memory = Dict{String, Any}(
:events=> Vector{Dict{Symbol, Any}}(), :events=> Vector{Dict{String, Any}}(),
:state=> Dict{Symbol, Any}(), # state of the agent :state=> Dict{String, Any}(), # state of the agent
:recap=> OrderedDict{Symbol, Any}(), # recap summary of the conversation :recap=> OrderedDict{String, Any}(), # recap summary of the conversation
) )
newAgent = companion( newAgent = companion(
@@ -91,89 +89,58 @@ function companion(
end end
""" A sommelier agent.
# Arguments
- `mqttClient::Client`
MQTTClient's client
- `msgMeta::Dict{Symbol, Any}`
A dict contain info about a message.
- `config::Dict{Symbol, Any}`
Config info for an agent. Contain mqtt topic for internal use and other info.
# Keyword Arguments
- `name::String`
Agent's name
- `id::String`
Agent's ID
- `tools::Dict{Symbol, Any}`
Agent's tools
- `maxHistoryMsg::Integer`
max history message
# Return
- `nothing`
# Example
```jldoctest
julia> using YiemAgent, MQTTClient, GeneralUtils
julia> msgMeta = GeneralUtils.generate_msgMeta(
"N/A",
replyTopic = "/testtopic/prompt"
)
julia> tools= Dict(
:chatbox=>Dict(
:name => "chatbox",
:description => "Useful only for when you need to ask the user for more info or context. Do not ask the user their own question.",
:input => "Input should be a text.",
:output => "" ,
:func => nothing,
),
)
julia> agentConfig = Dict(
:receiveprompt=>Dict(
:mqtttopic=> "/testtopic/prompt", # topic to receive prompt i.e. frontend send msg to this topic
),
:receiveinternal=>Dict(
:mqtttopic=> "/testtopic/internal", # receive topic for model's internal
),
:text2text=>Dict(
:mqtttopic=> "/text2text/receive",
),
)
julia> client, connection = MakeConnection("test.mosquitto.org", 1883)
julia> agent = YiemAgent.bsommelier(
client,
msgMeta,
agentConfig,
name= "assistant",
id= "555", # agent instance id
tools=tools,
)
```
# TODO
- [] update docstring
- [x] implement the function
# Signature
"""
mutable struct sommelier <: agent mutable struct sommelier <: agent
name::String # agent name name::String # agent name
id::String # agent id id::String # agent id
retailername::String retailername::String
tools::Dict tools::Dict
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{Symbol, Any}} chathistory::Vector{Dict{String, Any}}
memory::Dict{Symbol, Any} memory::Dict{String, Any}
context # NamedTuple of functions context::agentcontext
llmFormatName::String llmFormatName::String
end end
""" A sommelier agent.
# Arguments
- `context::agentcontext`
Application context containing shared functions for LLM, SQL, and vector database operations.
# Keyword Arguments
- `name::String`
Agent's name. Default: `"Assistant"`
- `id::String`
Agent's ID. Default: generated UUID string.
- `retailername::String`
Retailer name associated with the sommelier. Default: `"retailer_name"`
- `maxHistoryMsg::Integer`
Maximum history messages. Default: `20`
- `chathistory::Vector{Dict{Symbol, String}}`
Chat history. Default: empty vector.
- `llmFormatName::String`
LLM format name. Default: `"granite3"`
# Return
- `sommelier`: An instantiated sommelier agent.
# Example
```julia
julia> using YiemAgent
julia> context = agentcontext(
text2textInstructLLM,
getTextEmbedding,
executeSQL,
similarSQLVectorDB,
insertSQLVectorDB,
similarSommelierDecision,
insertSommelierDecision
)
julia> agent = sommelier(context, name="WineExpert", id="123", retailername="MyWineShop")
```
"""
function sommelier( function sommelier(
context::appcontext, # app context context::agentcontext, # app context
; ;
name::String= "Assistant", name::String= "Assistant",
id::String= string(uuid4()), id::String= string(uuid4()),
@@ -204,15 +171,15 @@ function sommelier(
Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()), Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()),
] ]
""" """
memory = Dict{Symbol, Any}( memory = Dict{String, Any}(
:shortmem=> OrderedDict{Symbol, Any}( :shortmem=> OrderedDict{String, Any}(
:db_search_result=> Any[], :db_search_result=> Any[],
:scratchpad=> "", #[PENDING] should be a dict e.g. Dict(:database_search_result=>Dict(:wines=> "", :search_query=> "")) :scratchpad=> "", #[PENDING] should be a dict e.g. Dict(:database_search_result=>Dict(:wines=> "", :search_query=> ""))
), ),
:events=> Vector{Dict{Symbol, Any}}(), :events=> Vector{Dict{String, Any}}(),
:state=> Dict{Symbol, Any}( :state=> Dict{String, Any}(
), ),
:recap=> OrderedDict{Symbol, Any}(), :recap=> OrderedDict{String, Any}(),
) )
@@ -238,8 +205,8 @@ mutable struct virtualcustomer <: agent
systemmsg::String # system message systemmsg::String # system message
tools::Dict tools::Dict
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{Symbol, Any}} chathistory::Vector{Dict{String, Any}}
memory::Dict{Symbol, Any} memory::Dict{String, Any}
context # NamedTuple of functions context # NamedTuple of functions
llmFormatName::String llmFormatName::String
end end
@@ -281,13 +248,13 @@ function virtualcustomer(
Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()), Dict(:name=>"assistant", :text=> "Hi I'm your assistant.", :timestamp=> Dates.now()),
] ]
""" """
memory = Dict{Symbol, Any}( memory = Dict{String, Any}(
:shortmem=> OrderedDict{Symbol, Any}( :shortmem=> OrderedDict{String, Any}(
), ),
:events=> Vector{Dict{Symbol, Any}}(), :events=> Vector{Dict{String, Any}}(),
:state=> Dict{Symbol, Any}( :state=> Dict{String, Any}(
), ),
:recap=> OrderedDict{Symbol, Any}(), :recap=> OrderedDict{String, Any}(),
) )
newAgent = virtualcustomer( newAgent = virtualcustomer(

45
src/util.jl
View File

@@ -60,6 +60,17 @@ end
""" Add new message to agent. """ Add new message to agent.
messages => Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Describe this image for me"),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data_uri)
)
]
)
Arguments\n Arguments\n
----- -----
a::agent a::agent
@@ -76,44 +87,24 @@ end
Example\n Example\n
----- -----
```jldoctest ```jldoctest
julia> using YiemAgent, MQTTClient, GeneralUtils
julia> client, connection = MakeConnection("test.mosquitto.org", 1883)
julia> connect(client, connection)
julia> msgMeta = GeneralUtils.generate_msgMeta("testtopic")
julia> agentConfig = Dict(
:receiveprompt=>Dict(
:mqtttopic=> "testtopic/receive",
),
:receiveinternal=>Dict(
:mqtttopic=> "testtopic/internal",
),
:text2text=>Dict(
:mqtttopic=> "testtopic/text2text",
),
)
julia> a = YiemAgent.sommelier(
client,
msgMeta,
agentConfig,
)
julia> YiemAgent.addNewMessage(a, "user", "hello")
``` ```
Signature\n Signature\n
----- -----
""" """
function addNewMessage(a::T1, name::String, text::T2; function addNewMessage(a::T1, name::String, userinput::T2;
maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractString} maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractDict}
if name ∉ ["system", "user", "assistant"] # guard against typo if name ∉ ["system", "user", "assistant"] # guard against typo
error("name is not in agent.availableRole $(@__LINE__)") error("name is not in agent.availableRole $(@__LINE__)")
end end
#[PENDING] summarize the oldest 10 message #TODO summarize the oldest 10 message
if length(a.chathistory) > maximumMsg if length(a.chathistory) > maximumMsg
summarize(a.chathistory) summarize(a.chathistory)
else else
d = Dict(:name=> name, :text=> text, :timestamp=> Dates.now()) userinput["timestamp"] = Dates.now()
push!(a.chathistory, d) push!(a.chathistory, userinput)
end end
end end
@@ -241,7 +232,7 @@ function eventdict(;
note::Union{String, Nothing}=nothing, note::Union{String, Nothing}=nothing,
) )
d = Dict{Symbol, Any}( d = Dict{String, Any}(
:event_description=> event_description, :event_description=> event_description,
:timestamp=> timestamp, :timestamp=> timestamp,
:subject=> subject, :subject=> subject,

2
test/prompttest_1.jl
View File

@@ -66,7 +66,7 @@ tools=Dict( # update input format
input = input =
OrderedDict{Symbol, Any}(:question => "Hello, I would like a get a bottle of wine", :thought_1 => "It's great that the user is looking for a bottle of wine. To give them a personalized recommendation, I need to know more about their preferences.", :action_1 => Dict{Symbol, Any}(:name => "chatbox", :input => "What occasion are you planning to use this wine for?"), :observation_1 => "We are holding a wedding party", :thought_2 => "A wedding party is a great occasion for a special bottle of wine. I need to know what type of food will be served, and how much the user is willing to spend.", :action_2 => Dict{Symbol, Any}(:name => "chatbox", :input => "What type of food will you be serving at the wedding?"), :observation_2 => "It will be Thai dishes.", :thought_3 => "The type of wine that pairs well with Thai dishes is usually a crisp and refreshing white wine, but I also need to consider the budget and personal preferences.", :action_3 => Dict{Symbol, Any}(:name => "chatbox", :input => "How much are you willing to spend on this bottle of wine?"), :observation_3 => "I would spend up to 50 bucks.", :thought_4 => "I have a good idea of the occasion, food, and budget. Now I need to know what type of wine the user is looking for.", :action_4 => Dict{Symbol, Any}(:name => "chatbox", :input => "What type of wine are you usually looking for? Red, White, Sparkling, Rose, Dessert or Fortified?"), :observation_4 => "I like full-bodied Red wine with low tannin.", :thought_5 => "Now that I have all the necessary information, I can start searching for a suitable wine in our inventory.", :action_5 => Dict{Symbol, Any}(:name => "winestock", :input => "red wine with low tannins"), :observation_5 => "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n", :thought_6 => "Now that I have the information about the wine, it's time to make a recommendation.", :action_6 => Dict{Symbol, Any}(:name => "recommendbox", :input => "El Enemigo Cabernet Franc 2019"), :observation_6 => "I don't like the one you recommend. I want dry wine.") OrderedDict{String, Any}(:question => "Hello, I would like a get a bottle of wine", :thought_1 => "It's great that the user is looking for a bottle of wine. To give them a personalized recommendation, I need to know more about their preferences.", :action_1 => Dict{String, Any}(:name => "chatbox", :input => "What occasion are you planning to use this wine for?"), :observation_1 => "We are holding a wedding party", :thought_2 => "A wedding party is a great occasion for a special bottle of wine. I need to know what type of food will be served, and how much the user is willing to spend.", :action_2 => Dict{String, Any}(:name => "chatbox", :input => "What type of food will you be serving at the wedding?"), :observation_2 => "It will be Thai dishes.", :thought_3 => "The type of wine that pairs well with Thai dishes is usually a crisp and refreshing white wine, but I also need to consider the budget and personal preferences.", :action_3 => Dict{String, Any}(:name => "chatbox", :input => "How much are you willing to spend on this bottle of wine?"), :observation_3 => "I would spend up to 50 bucks.", :thought_4 => "I have a good idea of the occasion, food, and budget. Now I need to know what type of wine the user is looking for.", :action_4 => Dict{String, Any}(:name => "chatbox", :input => "What type of wine are you usually looking for? Red, White, Sparkling, Rose, Dessert or Fortified?"), :observation_4 => "I like full-bodied Red wine with low tannin.", :thought_5 => "Now that I have all the necessary information, I can start searching for a suitable wine in our inventory.", :action_5 => Dict{String, Any}(:name => "winestock", :input => "red wine with low tannins"), :observation_5 => "I found the following wines in our stock: \n{\n 1: El Enemigo Cabernet Franc 2019\n2: Tantara Chardonnay 2017\n\n}\n", :thought_6 => "Now that I have the information about the wine, it's time to make a recommendation.", :action_6 => Dict{String, Any}(:name => "recommendbox", :input => "El Enemigo Cabernet Franc 2019"), :observation_6 => "I don't like the one you recommend. I want dry wine.")
result = YiemAgent.jsoncorrection(a, input) result = YiemAgent.jsoncorrection(a, input)

108
test/runtests.jl
View File

@@ -0,0 +1,108 @@
using JSON, Dates, UUIDs, PrettyPrinting, Base64, NATS, HTTP
using GeneralUtils, msghandler
config = JSON.parsefile("./appconfig.json")
agent_conn = NATS.connect(config["nats_server_info"]["url"])
function text2text_instruct_llm(openai_msg::AbstractDict)
payloads = [("msg", openai_msg, "dictionary")] # List of tuples
_, msg_envelope_json_str = msghandler.smartpack(
config["externalService"]["servicesloadbalancer"]["nats"],
payloads;
msg_purpose="text2text",
broker_url=config["nats_server_info"]["url"],
fileserver_url=config["externalService"]["fileserver"]["url"])
reply = NATS.request(agent_conn,
config["externalService"]["servicesloadbalancer"]["nats"],
msg_envelope_json_str, timeout=120)
incoming_env_json_str = String(reply.payload)
incoming_env = msghandler.smartunpack(incoming_env_json_str)
_llm_response = incoming_env["payloads"][1][2]
llm_response = _llm_response["choices"][1]["message"]["content"]
return llm_response
end
# 1. Read local file and encode to base64 string
image1_path = "test/large_image.png"
image1_bytes = read(image1_path)
image1_base64_string = base64encode(image1_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png"
data1_uri = "data:$(mime_type);base64,$(image1_base64_string)"
# 3. Construct payload with the Data URI
openai_msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => [
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Do you know this wine? Just give me brief intro."),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data1_uri)
)
]
)
],
"temperature" => 0.7
)
llm_response = text2text_instruct_llm(openai_msg)
# 1. Read local file and encode to base64 string
image2_path = "test/small_image.png"
image2_bytes = read(image2_path)
image2_base64_string = base64encode(image2_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png"
data2_uri = "data:$(mime_type);base64,$(image2_base64_string)"
openai_msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => [
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "Do you know this wine? Just give me brief intro."),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data1_uri)
)
]
),
Dict(
"role" => "assistant",
"content" => [
Dict("type" => "text", "text" => "Yes, I do!\n\nThis is **Asolo Bella Principessa**, a high-quality Italian sparkling wine made from the Prosecco region.\n\n### 🥂 Brief Intro\n\n* **What it is:** A Prosecco Superiore D.O.C.G., meaning it meets strict quality standards for a premium sparkling wine.\n* **Style:** It is a **Sparkling White Wine** and is designated as **Extra Dry**. This means it is crisp, refreshing, and has a dry finish (not overly sweet).\n* **Flavor Profile:** Expect bright, lively bubbles, often with notes of green apple, pear, and citrus.\n* **Best For:** It's a versatile wine, perfect for celebratory toasts, enjoying with appetizers (like seafood or charcuterie), or simply as a refreshing aperitivo."),
]
),
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "How does this wine differ from earlier wine?"),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data2_uri)
)
]
),
],
"temperature" => 0.7
)
llm_response = text2text_instruct_llm(openai_msg)