ton/YiemAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-06-27 08:00:41 +07:00
parent 299a485e4e
commit e63dd7d898
8 changed files with 534 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Manifest.toml
+52 -3
View File
@@ -2,7 +2,7 @@
julia_version = "1.12.6" julia_version = "1.12.6"
manifest_format = "2.0" manifest_format = "2.0"
project_hash = "7462b22f4fd62982e36c8671793df6d8908c9ad0" project_hash = "972fb718a7c47dc6826419825a1ab5fe7b0fde84"
[[deps.AliasTables]] [[deps.AliasTables]]
deps = ["PtrArrays", "Random"] deps = ["PtrArrays", "Random"]
@@ -14,6 +14,12 @@ version = "1.1.3"
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
version = "1.1.2" version = "1.1.2"
[[deps.Arrow]]
deps = ["ArrowTypes", "BitIntegers", "CodecLz4", "CodecZstd", "ConcurrentUtilities", "DataAPI", "Dates", "EnumX", "Mmap", "PooledArrays", "SentinelArrays", "StringViews", "Tables", "TimeZones", "TranscodingStreams", "UUIDs"]
git-tree-sha1 = "4a69a3eadc1f7da78d950d1ef270c3a62c1f7e01"
uuid = "69666777-d1a9-59fb-9406-91d4454c9d45"
version = "2.8.1"
[[deps.ArrowTypes]] [[deps.ArrowTypes]]
deps = ["Sockets", "UUIDs"] deps = ["Sockets", "UUIDs"]
git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101" git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101"
@@ -28,6 +34,12 @@ version = "1.11.0"
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
version = "1.11.0" version = "1.11.0"
[[deps.BitIntegers]]
deps = ["Random"]
git-tree-sha1 = "091d591a060e43df1dd35faab3ca284925c48e46"
uuid = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1"
version = "0.3.7"
[[deps.BufferedStreams]] [[deps.BufferedStreams]]
git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643" git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643"
uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
@@ -60,12 +72,24 @@ git-tree-sha1 = "40956acdbef3d8c7cc38cba42b56034af8f8581a"
uuid = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf" uuid = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf"
version = "0.3.4" version = "0.3.4"
[[deps.CodecLz4]]
deps = ["Lz4_jll", "TranscodingStreams"]
git-tree-sha1 = "d58afcd2833601636b48ee8cbeb2edcb086522c2"
uuid = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
version = "0.4.6"
[[deps.CodecZlib]] [[deps.CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"] deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193" uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.8" version = "0.7.8"
[[deps.CodecZstd]]
deps = ["TranscodingStreams", "Zstd_jll"]
git-tree-sha1 = "da54a6cd93c54950c15adf1d336cfd7d71f51a56"
uuid = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
version = "0.8.7"
[[deps.Compat]] [[deps.Compat]]
deps = ["TOML", "UUIDs"] deps = ["TOML", "UUIDs"]
git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad"
@@ -86,6 +110,12 @@ deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
version = "1.3.0+1" version = "1.3.0+1"
[[deps.ConcurrentUtilities]]
deps = ["Serialization", "Sockets"]
git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8"
uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb"
version = "2.5.1"
[[deps.Crayons]] [[deps.Crayons]]
git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
@@ -424,6 +454,12 @@ git-tree-sha1 = "0aad96d7b987a5600e260eec50147b254d5ff7e6"
uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
version = "3.6.0" version = "3.6.0"
[[deps.Lz4_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "191686b1ac1ea9c89fc52e996ad15d1d241d1e33"
uuid = "5ced341a-0733-55b8-9ab6-a4889d929147"
version = "1.10.1+0"
[[deps.Markdown]] [[deps.Markdown]]
deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -662,7 +698,7 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0" version = "0.7.0"
[[deps.SQLLLM]] [[deps.SQLLLM]]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "HTTP", "JSON3", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"] deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "GeneralUtils", "HTTP", "JSON3", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"]
path = "../SQLLLM" path = "../SQLLLM"
uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3" uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
version = "0.2.4" version = "0.2.4"
@@ -769,6 +805,11 @@ git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e"
version = "0.4.4" version = "0.4.4"
[[deps.StringViews]]
git-tree-sha1 = "f2dcb92855b31ad92fe8f079d4f75ac57c93e4b8"
uuid = "354b36f9-a18e-4713-926e-db85100087ba"
version = "1.3.7"
[[deps.StructTypes]] [[deps.StructTypes]]
deps = ["Dates", "UUIDs"] deps = ["Dates", "UUIDs"]
git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8"
@@ -884,7 +925,7 @@ uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60"
version = "1.6.1" version = "1.6.1"
[[deps.YiemAgent]] [[deps.YiemAgent]]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "Serialization", "URIs", "UUIDs"] deps = ["CSV", "DataFrames", "DataStructures", "Dates", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SQLLLM", "Serialization", "URIs", "UUIDs"]
path = "." path = "."
uuid = "e012c34b-7f78-48e0-971c-7abb83b6f0a2" uuid = "e012c34b-7f78-48e0-971c-7abb83b6f0a2"
version = "0.4.0" version = "0.4.0"
@@ -911,6 +952,14 @@ git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026"
uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8"
version = "1.0.21+0" version = "1.0.21+0"
[[deps.msghandler]]
deps = ["Arrow", "Base64", "DataFrames", "Dates", "GeneralUtils", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"]
git-tree-sha1 = "02825c9cb2c15d3bc30fdf28656c1f594a04cb5f"
repo-rev = "main"
repo-url = "https://git.yiem.cc/ton/msghandler"
uuid = "f2724d33-f338-4a57-b9f8-1be882570d10"
version = "0.5.6"
[[deps.nghttp2_jll]] [[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"] deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
Project.toml
+2
View File
@@ -21,6 +21,7 @@ SQLLLM = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
msghandler = "f2724d33-f338-4a57-b9f8-1be882570d10"
[compat] [compat]
CSV = "0.10.15" CSV = "0.10.15"
@@ -29,3 +30,4 @@ GeneralUtils = "0.4.0"
HTTP = "2.4.0" HTTP = "2.4.0"
JSON = "1.6.1" JSON = "1.6.1"
NATS = "0.1.0" NATS = "0.1.0"
msghandler = "0.5.6"
dummy_config.json
+53
View File
@@ -0,0 +1,53 @@
{
"nats_server_info": {
"description": "nats server",
"url": "nats.yiem.cc"
},
"testingOrProduction": "testing",
"agentId": "2b74b87a-5413-4fe2-a4d3-405891051680",
"agentCentralConfigSubject": "/yiem/hq/agent/sommelier/backend/config/api/v1.1",
"this_service_input_channel": {
"mqtt": [
"/yiem/hq/agent/sommpanion/backend/db/api_v1"
],
"nats": [
"sommpanion.backend.agentbackend.v1.inbox"
]
},
"agentRole": "sommelier",
"organization": "yiem_hq",
"externalService": {
"servicesloadbalancer": {
"nats": "sommpanion.backend.servicesloadbalancer.v1.inbox"
},
"textembedding": {
"url": "textembedding.api.v1"
},
"textimage_to_text_llm": {
"url": "https://llmcoder.yiem.cc/v1/chat/completions",
"modelname": "Qwen3.6-35B-A3B-UD-Q4_K_M"
},
"virtualWineCustomer_1": {
"serviceSubject": "",
"modelName": "qwen3:8b"
},
"sommpanion_db" : {
"description": "A database connection info for LibPQ client",
"url": "192.168.88.106:5432",
"dbname": "winedb",
"user": "yiemtechnologies@gmail.com",
"password": "yiemtechnologies@Postgres_0.0"
},
"sommpanion_vectordb" : {
"description": "A wine database connection info for LibPQ client",
"url": "192.168.88.106:5433",
"dbname": "vectordb",
"user": "yiemtechnologies@gmail.com",
"password": "yiemtechnologies@Postgres_0.0"
},
"fileserver": {
"description": "temporary file server",
"url": "https://fileserver.yiem.cc"
}
}
}
src/interface.jl
+128 -421
View File
@@ -122,21 +122,21 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
# """ # """
# end # end
recentevents_ind = GeneralUtils.recentElementsIndex( # recentevents_ind = GeneralUtils.recentElementsIndex(
length(a.memory["events"]), recentevents; includelatest=true) # length(a.memory["events"]), recentevents; includelatest=true)
requiredKeys = ["plan", "actionname", "actioninput"] requiredKeys = ["plan", "actionname", "actioninput"]
context = context =
""" """
<internal_context_for_assistant> <internal_context_for_assistant>
$(a.memory["scratchpad"]) $(a.memory["shortmem"]["scratchpad"])
</internal_context_for_assistant> </internal_context_for_assistant>
""" """
#WORKING add context to text of the latest message (in the front). # add context to text of the latest message (in the front).
# use for loop because in openai format, each msg may contain both text and image. # use for loop because in openai format, each msg may contain both text and image.
for d in enumerate(a.chathistory[end]["content"]) for d in a.chathistory[end]["content"]
if d["type"] == "text" if d["type"] == "text"
d["text"] = context * d["text"] d["text"] = context * d["text"]
break break
@@ -145,28 +145,27 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
errornote = "N/A" errornote = "N/A"
response = nothing # placeholder for show when error msg show up response = nothing # placeholder for show when error msg show up
for attempt in 1:maxattempt for attempt in 1:maxattempt
if attempt > 1 if attempt > 1
println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end end
openai_msg = Dict( openai_msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL", "model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => a.chathistory, "messages" => a.chathistory,
"temperature" => 0.7 "temperature" => 0.7
) )
response = a.context.text2textInstructLLM(a.id, openai_msg)
response = a.context.text2textInstructLLM(prompt; senderId=a.id) # response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)
response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)
response = GeneralUtils.remove_french_accents(response) response = GeneralUtils.remove_french_accents(response)
think, response = GeneralUtils.extractthink(response) think, response = GeneralUtils.extractthink(response)
response = String(split(response, ", observation")[1]) # in case LLM generate observation key which it isn't supposed to response = String(split(response, ", observation")[1]) # in case LLM generate observation key which it isn't supposed to
response = strip(response) response = strip(response)
responsedict = nothing responsedict = nothing
try try
responsedict = copy(JSON.parsefile(response)) _responsedict = JSON.parse(response)
responsedict = GeneralUtils.dictify(_responsedict, keytype=String)
catch catch
println("\nERROR YiemAgent decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") println("\nERROR YiemAgent decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue continue
@@ -179,21 +178,6 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
continue continue
end end
# _responsedictKey = keys(responsedict)
# responsedictKey = [i for i in _responsedictKey] # convert into a list
# is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys]
# if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys)
# errornote = "Your previous attempt has more key points than answer's required key points."
# println("\nERROR YiemAgent decisionMaker() $errornote ----(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# continue
# elseif !all(is_requiredKeys_in_responsedictKey)
# zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey)
# missingkeys = [requiredKeys[i] for i in zeroind]
# errornote = "$missingkeys are missing from your previous response"
# println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# continue
# end
if responsedict["actionname"] ∉ ["CHATBOX", "CHECKWINE", "PRESENTBOX", "ENDCONVERSATION"] if responsedict["actionname"] ∉ ["CHATBOX", "CHECKWINE", "PRESENTBOX", "ENDCONVERSATION"]
errornote = "Your previous attempt didn't use the given functions" errornote = "Your previous attempt didn't use the given functions"
@@ -204,64 +188,64 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
println("\nYiem decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") println("\nYiem decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict)) pprintln(Dict(responsedict))
# check whether an agent recommend wines before checking inventory or recommend wines # # check whether an agent recommend wines before checking inventory or recommend wines
# outside its inventory # # outside its inventory
# ask LLM whether there are any winery mentioned in the response # # ask LLM whether there are any winery mentioned in the response
mentioned_winery = detectWineryName(a, response) # mentioned_winery = detectWineryName(a, response)
if mentioned_winery != "None" # if mentioned_winery != "None"
mentioned_winery = String.(strip.(split(mentioned_winery, ","))) # mentioned_winery = String.(strip.(split(mentioned_winery, ",")))
# check whether the wine is in event # # check whether the wine is in event
isWineInEvent = false # isWineInEvent = false
for winename in mentioned_winery # for winename in mentioned_winery
for event in a.memory["events"] # for event in a.memory["events"]
if event["observation"] !== nothing && occursin(winename, event["observation"]) # if event["observation"] !== nothing && occursin(winename, event["observation"])
isWineInEvent = true # isWineInEvent = true
break # break
end # end
end # end
end # end
# then the agent is not supposed to recommend the wine # # then the agent is not supposed to recommend the wine
if isWineInEvent == false # if isWineInEvent == false
errornote = "You recommended wines that are not in your inventory before. Please only recommend wines that you have previously found in your inventory." # errornote = "You recommended wines that are not in your inventory before. Please only recommend wines that you have previously found in your inventory."
println("\nERROR YiemAgent decisionMaker() $errornote $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println("\nERROR YiemAgent decisionMaker() $errornote $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue # continue
end # end
end # end
delete!(responsedict, :mentioned_winery) # delete!(responsedict, :mentioned_winery)
# check whether responsedict["actioninput"] is the same as previous dialogue # check whether responsedict["actioninput"] is the same as previous dialogue
if !isempty(a.chathistory) && responsedict["actioninput"] == a.chathistory[end]["text"] # if !isempty(a.chathistory) && responsedict["actioninput"] == a.chathistory[end]["text"]
errornote = "In your previous attempt, you repeated the previous dialogue. Please try again." # errornote = "In your previous attempt, you repeated the previous dialogue. Please try again."
println("\nERROR YiemAgent decisionMaker() $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println("\nERROR YiemAgent decisionMaker() $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue # continue
end # end
evaluationdict = evaluator(a, timeline, responsedict, context) # evaluationdict = evaluator(a, timeline, responsedict, context)
if evaluationdict[:approval] == "no" # if evaluationdict[:approval] == "no"
mentor_comment = evaluationdict[:suggestion] # mentor_comment = evaluationdict[:suggestion]
errornote = "Your previous attempt was not good enough. Please try again. Here is the mentor's suggestion: $mentor_comment" # errornote = "Your previous attempt was not good enough. Please try again. Here is the mentor's suggestion: $mentor_comment"
println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)--> \n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)--> \n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue # continue
end # end
# store for later training # # store for later training
responsedict["system"] = systemmsg # responsedict["system"] = systemmsg
responsedict["unformatPrompt"] = unformatPrompt # responsedict["unformatPrompt"] = unformatPrompt
responsedict["prompt"] = prompt # responsedict["prompt"] = prompt
responsedict["context"] = context # responsedict["context"] = context
responsedict["think"] = think # responsedict["think"] = think
responsedict["response"] = response # responsedict["response"] = response
# responsedict["QandA"] = QandA # # responsedict["QandA"] = QandA
# check whether there is a file path exists before writing to it # # check whether there is a file path exists before writing to it
if !haskey(a.memory["shortmem"], "decisionlog") # if !haskey(a.memory["shortmem"], "decisionlog")
a.memory["shortmem"]["decisionlog"] = [responsedict] # a.memory["shortmem"]["decisionlog"] = [responsedict]
else # else
push!(a.memory["shortmem"]["decisionlog"], responsedict) # push!(a.memory["shortmem"]["decisionlog"], responsedict)
end # end
# # save to filename ./log/decisionlog.txt # # save to filename ./log/decisionlog.txt
# println("\nsaving YiemAgent decisionMaker() to disk ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println("\nsaving YiemAgent decisionMaker() to disk ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
@@ -286,7 +270,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
# println("\nYiemAgent decisionMaker() saved to disk is done. agent $(a.id)") # println("\nYiemAgent decisionMaker() saved to disk is done. agent $(a.id)")
responsedict["prompt"] = prompt # responsedict["prompt"] = prompt
return responsedict return responsedict
end end
error("DecisionMaker failed to generate a thought ", response) error("DecisionMaker failed to generate a thought ", response)
@@ -468,7 +452,7 @@ base64_string = base64encode(image_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg) # 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png" mime_type = "image/png"
data_uri = "data:$(mime_type);base64,$(base64_string)" data1_uri = "data:<mime_type>;base64,<image1_base64_string>"
# 3. Construct payload with the Data URI # 3. Construct payload with the Data URI
message => Dict( message => Dict(
@@ -483,33 +467,31 @@ message => Dict(
) )
# ---------------------------------------------- 100 --------------------------------------------- # # ---------------------------------------------- 100 --------------------------------------------- #
""" #WORKING """
function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Object{String, Any}}, maximumMsg=50) function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Object{String, Any}},
maximumMsg=50)
userinput = GeneralUtils.dictify(userinput; keytype=String) userinput = GeneralUtils.dictify(userinput; keytype=String)
# find text in usermsg
usertext = nothing
text_position = nothing
for (i, d) in enumerate(userinput["content"])
if d["type"] == "text"
usertext = d["text"]
text_position = i
end
end
# place holder # place holder
actionname = nothing actionname = nothing
result = nothing result = nothing
chatresponse = nothing chatresponse = nothing
if userinput === nothing if usertext == "newtopic"
# thinking loop until AI wants to communicate with the user
chatresponse = nothing
while chatresponse === nothing
actionname, result = think(a)
if actionname ∈ ["CHATBOX", "PRESENTBOX", "ENDCONVERSATION"]
chatresponse = result
end
end
addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg)
return chatresponse
elseif user_text_input == "newtopic"
clearhistory(a) clearhistory(a)
return "Okay. What shall we talk about?" return "Okay. What shall we talk about?"
else else
userinput["content"][text_position] = GeneralUtils.remove_french_accents(user_text_input) userinput["content"][text_position]["text"] = GeneralUtils.remove_french_accents(usertext)
# add usermsg to a.chathistory but how do I handle images? # add usermsg to a.chathistory but how do I handle images?
addNewMessage(a, "user", userinput; maximumMsg=maximumMsg) addNewMessage(a, "user", userinput; maximumMsg=maximumMsg)
@@ -521,7 +503,11 @@ function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Obj
chatresponse = result chatresponse = result
end end
end end
addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg) assistant_response = Dict{String, Any}(
"role" => "assistant",
"content" => [Dict("type" => "text", "text" => chatresponse),]
)
addNewMessage(a, "assistant", assistant_response; maximumMsg=maximumMsg)
return chatresponse return chatresponse
end end
@@ -568,7 +554,6 @@ function conversation(a::Union{companion, virtualcustomer}, userinput::Dict;
end end
""" """
# Arguments # Arguments
# Return # Return
@@ -578,92 +563,49 @@ end
julia> julia>
``` ```
""" # WORKING """
function think(a::T)::namedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent} function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent}
# a.memory[:recap] = generateSituationReport(a, a.context["text"2textInstructLLM]; skiprecent=0) # a.memory[:recap] = generateSituationReport(a, a.context["text"2textInstructLLM]; skiprecent=0)
thoughtDict = decisionMaker(a) thoughtDict = decisionMaker(a)
actionname = thoughtDict["actionname"] # # map action and input() to llm function
actioninput = thoughtDict["actioninput"] # response =
# if thoughtDict["actionname"] == "CHATBOX" || thoughtDict["actionname"] == "ENDCONVERSATION"
# (result=thoughtDict["plan"], errormsg=nothing, success=true)
# elseif thoughtDict["actionname"] == "CHECKWINE"
# checkwine(a, thoughtDict["actioninput"])
# elseif thoughtDict["actionname"] == "PRESENTBOX"
# (result=thoughtDict["actioninput"], errormsg=nothing, success=true)
# else
# error("undefined LLM function. Requesting $(thoughtDict["actionname"])")
# end
# map action and input() to llm function # # this section allow LLM functions above to have different return values.
response = # result = haskey(response, "result") ? response["result"] : nothing
if actionname == "CHATBOX" || actionname == "ENDCONVERSATION" # rawresponse = haskey(response, "rawresponse") ? response["rawresponse"] : nothing
(result=thoughtDict["plan"], errormsg=nothing, success=true) # select = haskey(response, "select") ? response["select"] : nothing
elseif actionname == "CHECKWINE" # reward::Integer = haskey(response, "reward") ? response["reward"] : 0
checkwine(a, actioninput) # isterminal::Bool = haskey(response, "isterminal") ? response["isterminal"] : false
elseif actionname == "PRESENTBOX" # errormsg::Union{AbstractString,Nothing} = haskey(response, "errormsg") ? response["errormsg"] : nothing
(result=actioninput, errormsg=nothing, success=true) # success::Bool = haskey(response, "success") ? response["success"] : false
# elseif actionname == "ENDCONVERSATION"
# x = "Conclude the conversation, thanks the user then goodbye and inviting them to return next time."
# (result=actioninput, errormsg=nothing, success=true)
else
error("undefined LLM function. Requesting $actionname")
end
# this section allow LLM functions above to have different return values. result = nothing
result = haskey(response, "result") ? response["result"] : nothing if thoughtDict["actionname"] ∈ ["CHATBOX"]
rawresponse = haskey(response, "rawresponse") ? response["rawresponse"] : nothing result = thoughtDict["actioninput"]
select = haskey(response, "select") ? response["select"] : nothing elseif thoughtDict["actionname"] ∈ ["ENDCONVERSATION"]
reward::Integer = haskey(response, "reward") ? response["reward"] : 0 # WORKING add ENDCONVERSATION guideline in to scratchpad
isterminal::Bool = haskey(response, "isterminal") ? response["isterminal"] : false guideline =
errormsg::Union{AbstractString,Nothing} = haskey(response, "errormsg") ? response["errormsg"] : nothing """
success::Bool = haskey(response, "success") ? response["success"] : false To end conversation with the user
"""
# # manage memory (pass msg to generatechat) a.memory["shortmem"]["scratchpad"] =
# if actionname ∈ ["CHATBOX", "PRESENTBOX", "ENDCONVERSATION"]
# chatresponse = generatechat(a, thoughtDict)
# push!(a.memory["events"],
# eventdict(;
# event_description="the assistant talks to the user.",
# timestamp=Dates.now(),
# subject="assistant",
# thought=thoughtDict,
# actionname=actionname,
# actioninput=actioninput,
# )
# )
# result = chatresponse
if actionname ∈ ["CHATBOX"]
push!(a.memory["events"],
eventdict(;
event_description="an assistant talks to the user.",
timestamp=Dates.now(),
subject="assistant",
thought=thoughtDict,
actionname=actionname,
actioninput=actioninput,
)
)
result = actioninput
elseif actionname ∈ ["ENDCONVERSATION"]
chatresponse = generatechat(a, thoughtDict)
push!(a.memory["events"],
eventdict(;
event_description="an assistant talks to the user to end conversation.",
timestamp=Dates.now(),
subject="assistant",
thought=thoughtDict,
actionname=actionname,
actioninput=chatresponse,
)
)
result = chatresponse result = chatresponse
elseif actionname ∈ ["PRESENTBOX"] elseif thoughtDict["actionname"] ∈ ["PRESENTBOX"]
chatresponse = presentbox(a, thoughtDict) chatresponse = presentbox(a, thoughtDict) #PENDING
push!(a.memory["events"],
eventdict(;
event_description="the assistant presents wines to the user.",
timestamp=Dates.now(),
subject="assistant",
thought=thoughtDict,
actionname=actionname,
actioninput=chatresponse,
)
)
result = chatresponse result = chatresponse
elseif actionname == "CHECKWINE" elseif thoughtDict["actionname"] == "CHECKWINE"
if rawresponse !== nothing if rawresponse !== nothing
vd = GeneralUtils.dfToVectorDict(rawresponse) # comes in dataframe format vd = GeneralUtils.dfToVectorDict(rawresponse) # comes in dataframe format
# a.memory["shortmem"]["found_wine"] = vd # used by decisionMaker() as a short note # a.memory["shortmem"]["found_wine"] = vd # used by decisionMaker() as a short note
@@ -689,24 +631,11 @@ function think(a::T)::namedTuple{(:actionname, :result),Tuple{String,String}} wh
# </database_search_result> # </database_search_result>
# """ # """
end end
push!(a.memory["events"],
eventdict(;
event_description= "the assistant searched the database.",
timestamp= Dates.now(),
subject= "assistant",
thought=thoughtDict,
actionname=actionname,
actioninput= "I search the database with this search term: $actioninput",
observation= "This is what I found:, $result"
)
)
else else
error("condition is not defined ", @__FILE__, ":", @__LINE__, " $(Dates.now())") error("condition is not defined ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end end
return (actionname=actionname, result=result) return (actionname=thoughtDict["actionname"], result=result)
end end
@@ -866,94 +795,22 @@ function presentbox(a::sommelier, thoughtDict; maxtattempt::Integer=10, recentev
error("presentbox() failed to generate a response") error("presentbox() failed to generate a response")
end end
function endconversation(a::sommelier, thoughtDict; maxattempt::Integer=10)
""" text =
# Arguments
- `a::T1`
one of ChatAgent's agent.
- `input::T2`
# Return
A JSON string of available wine
# Example
```jldoctest
julia>
```
# TODO
- [] update docs
# Signature
"""
function generatechat(a::sommelier, thoughtDict; maxattempt::Integer=10)
systemmsg =
""" """
Your role: ---
Your name is $(a.name). You are a helpful English-speaking assistant, acting as a polite, website-based sommelier for $(a.retailername)'s wine store.
Situation:
You have some thinking in mind while you are talking with the user.
Your mission:
Concentrate on your thoughts and articulate them clearly. Keep the conversation engaging.
Your responsibility does NOT includes:
- Requesting the user to place an order, make a purchase, or confirm the order. These are the job of our sales team at the store.
- Processing sales orders or engaging in any other sales-related activities. These are the job of our sales team at the store.
- Answering questions or offering additional services beyond those related to your store's wine recommendations such as discounts, quantity, rewards programs, promotions, delivery options, shipping, boxes, gift wrapping, packaging, personalized messages or something similar. These are the job of our sales team at the store.
At each round of conversation, you will be given the following:
Your ongoing conversation with the user: ...
Your thoughts: Your current thoughts in your mind
You must follow the following guidelines:
- Do not offer additional services you didn't think
You should follow the following guidelines:
- Focus on the latest conversation
- If the user interrupts, prioritize the user
- Be honest
You should then respond to the user with:
dialogue: what you want to say to the user
You should only respond in JSON format as described below:
{
"dialogue": "..."
}
Here are some examples:
Your ongoing conversation with the user: "user> hello, I need a new car\n"
Your thoughts: "I should recommend the car we have found in our inventory to the user."
{"dialogue": "We have a variety of cars available, including the Toyota Camry 2020, the Honda Civic 2021, and the Ford Mustang 2022. Which one would you like to see?"}
Let's begin!
""" """
requiredKeys = [:dialogue] requiredKeys = ["dialogue"]
# a.memory["shortmem"]["available_wine"] is a vector of dictionary system_msg = Dict(
# context = "role" => "system",
# if length(a.memory["shortmem"]["available_wine"]) != 0 "content" => [
# "Wines previously found in your inventory: $(availableWineToText(a.memory["shortmem"]["available_wine"]))" Dict("type" => "text", "text" => systemmsg),
# else ]
# "N/A" )
# end
chathistory = chatHistoryToText(a.chathistory)
errornote = "N/A"
response = nothing # placeholder for show when error msg show up
yourthought = thoughtDict["plan"]
# yourthought1 = nothing
for attempt in 1:maxattempt for attempt in 1:maxattempt
# if attempt > 1 # use to prevent LLM generate the same respond over and over
# println("\nYiemAgent generatchat() attempt $attempt/10 ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# yourthought1 = paraphrase(a.context["text"2textInstructLLM], yourthought)
# else
# yourthought1 = yourthought
# end
context =
"""
<context>
Your ongoing conversation with the user: $chathistory
Your thoughts: $yourthought
P.S. $errornote
</context>
"""
unformatPrompt = unformatPrompt =
[ [
@@ -1036,156 +893,6 @@ function generatechat(a::sommelier, thoughtDict; maxattempt::Integer=10)
end end
function generatechat(a::companion; recentevents::Integer=10,
converPartnerName::Union{String, Nothing}=nothing, maxattempt=10)
recentchat_ind = GeneralUtils.recentElementsIndex(length(a.chathistory), recentevents;
includelatest=true);
recentchat = createChatLog(a.chathistory; index=recentchat_ind)
response = nothing # placeholder for show when error msg show up
errornote = "N/A"
for attempt in 1:maxattempt
if attempt > 1
println("\nYiemAgent generatechat() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
context =
"""
<context>
P.S. $errornote
</context>
"""
unformatPrompt =
[
Dict("name" => "system", "text" => a.systemmsg),
]
unformatPrompt = vcat(unformatPrompt, recentchat)
# put in model format
prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName)
# add info
prompt = prompt * context
# replace user and assistant with partner name
prompt = replace(_prompt, "|>user"=>"|>$(converPartnerName)")
prompt = replace(prompt, "|>assistant"=>"|>$(a.name)")
response = a.context.text2textInstructLLM(prompt; llmkwargs=llmkwargs, senderId=a.id)
response = replace(response, "<|im_start|>"=> "")
response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)
think, response = GeneralUtils.extractthink(response)
# check whether LLM just repeat the previous dialogue
for msg in a.chathistory
if msg["text"] == response
errornote = "In your previous attempt, you repeated the previous dialogue. Please try again."
println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
end
return response
end
error("generatechat failed to generate a response")
end
# modify it to work with customer object
function generatechat(a::virtualcustomer;
converPartnerName::Union{String, Nothing}=nothing, maxattempt=10, recentEventNum=10
)
recent_ind = GeneralUtils.recentElementsIndex(length(a.memory["events"]), recentEventNum; includelatest=true)
recentEventsDict = createEventsLog(a.memory["events"]; index=recent_ind)
response = nothing # placeholder for show when error msg show up
errornote = "N/A"
header = ["Dialogue:", "Role:"]
dictkey = ["dialogue", "role"]
llmkwargs=Dict(
"num_ctx" => 32768,
"temperature" => 0.5,
)
for attempt in 1:maxattempt
if attempt > 1
println("\nYiemAgent generatechat() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
context =
"""
<context>
P.S. $errornote
</context>
"""
unformatPrompt =
[
Dict("name" => "system", "text" => a.systemmsg),
]
unformatPrompt = vcat(unformatPrompt, recentEventsDict)
# put in model format
prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName)
# add info
prompt = prompt * context
response = a.context.text2textInstructLLM(prompt; llmkwargs=llmkwargs, senderId=a.id)
response = replace(response, "<|im_start|>"=> "")
response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)
think, response = GeneralUtils.extractthink(response)
# check whether response has all header
detected_kw = GeneralUtils.detectKeywordVariation(header, response)
missingkeys = [k for (k, v) in detected_kw if v === nothing]
if !isempty(missingkeys)
errornote = "$missingkeys are missing from your previous response"
println("\nERROR YiemAgent rolegenerator() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
elseif sum([length(i) for i in values(detected_kw)]) > length(header)
errornote = "\nYour previous attempt has duplicated points according to the required response format"
println("\nERROR YiemAgent rolegenerator() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
responsedict = GeneralUtils.textToDict(response, header;
dictKey=dictkey, symbolkey=true)
if responsedict["role"] == "no"
errornote = "In your previous attempt you said $(responsedict["dialogue"]) which you, as a customer of a wine store, are not supposed to speak."
println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
# check if the dialogue is the same as the previous one
if length(responsedict["dialogue"]) != 0 && responsedict["dialogue"] == a.chathistory[end]["text"]
errornote = "In your previous attempt you said $(responsedict["dialogue"]) which was the same as the previous one."
println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
# check whether LLM just repeat the previous dialogue
dublicate = false
for msg in a.chathistory
if msg["text"] == responsedict["dialogue"]
errornote = "In your previous attempt, you repeated the earlier dialogue. Please try again."
println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
dublicate = true
break
end
end
if dublicate
continue
end
# println("\n$prompt", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# println("\n $response")
return responsedict["dialogue"]
end
error("generatechat failed to generate a response")
end
function generatequestion(a, text2textInstructLLM::Function, timeline)::String function generatequestion(a, text2textInstructLLM::Function, timeline)::String
systemmsg = systemmsg =
""" """
src/type.jl
+4 -4
View File
@@ -171,7 +171,7 @@ function sommelier(
image1_bytes = read(image1_path) | this part must be done image1_bytes = read(image1_path) | this part must be done
image1_base64_string = base64encode(image1_bytes) | in frontend image1_base64_string = base64encode(image1_bytes) | in frontend
mime_type = "image/png" | not in agent code mime_type = "image/png" | not in agent code
data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" --- data1_uri = "data:<mime_type>;base64,<image1_base64_string>" ---
chathistory= [ chathistory= [
Dict( Dict(
@@ -193,7 +193,7 @@ function sommelier(
"image_url" => Dict("url" => data1_uri) "image_url" => Dict("url" => data1_uri)
), ),
] ]
) ),
] ]
""" """
memory = Dict{String, Any}( memory = Dict{String, Any}(
@@ -249,8 +249,8 @@ function sommelier(
Example query 1: "Dry, full-bodied red wine from 1) region: Burgundy, country: France or 2) region: Tuscany, country: Italy. Grape varietal: Merlot or Syrah. price 100 to 1000 USD." Example query 1: "Dry, full-bodied red wine from 1) region: Burgundy, country: France or 2) region: Tuscany, country: Italy. Grape varietal: Merlot or Syrah. price 100 to 1000 USD."
Example query 2: "Red or white wine, medium tannin, price under 700 USD" Example query 2: "Red or white wine, medium tannin, price under 700 USD"
Example query 3: "white wine, region: Tuscany or Bordeaux, country: Italy or France Example query 3: "white wine, region: Tuscany or Bordeaux, country: Italy or France
- PRESENTBOX which you can use to present wines you have found in your inventory to the user. The input are wine names that you want to present. The output is presentation of the wines. - PRESENTBOX which you can use to to generate proper dialogue to present wines you have found in your inventory to the user. The input are wine names that you want to present. The output is presentation of the wines.
- ENDCONVERSATION which you can use to properly end the conversation with the user. Input is a dialogue where you wrap up the conversation, thank the user, and invite them to return next time. - ENDCONVERSATION which you can use to generates a natural dialogue to gracefully close the conversation with the user. The input is "nothing" keyword. The output is a natural, context-aware dialogue extension that smoothly concludes the conversation.
</Available tools> </Available tools>
<your role> <your role>
Your name is $(newAgent.name). You are a sommelier for website-based $(newAgent.retailername)'s wine store. You are working under your mentor supervision. Your name is $(newAgent.name). You are a sommelier for website-based $(newAgent.retailername)'s wine store. You are working under your mentor supervision.
src/util.jl
+4 -4
View File
@@ -35,7 +35,7 @@ julia> agentConfig = Dict(
"text2text"=>Dict( "text2text"=>Dict(
"mqtttopic"=> "testtopic/text2text", "mqtttopic"=> "testtopic/text2text",
), ),
) )
julia> a = YiemAgent.sommelier( julia> a = YiemAgent.sommelier(
client, client,
msgMeta, msgMeta,
@@ -301,7 +301,7 @@ function createTimeline(events::T1; eventindex::Union{UnitRange, Nothing}=nothin
timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n" timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n"
else else
timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"])\\n" timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"])\\n"
end end
end end
# Return formatted timeline string # Return formatted timeline string
@@ -328,7 +328,7 @@ end
# timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: Not done yet.\n" # timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: Not done yet.\n"
# # If outcome exists, include it in formatting # # If outcome exists, include it in formatting
# else # else
# timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n" # timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n"
# end # end
# end # end
@@ -422,7 +422,7 @@ end
function checkAgentResponse_JSON(responsedict::Dict, requiredKeys::T function checkAgentResponse_JSON(responsedict::Dict, requiredKeys::T
)::Tuple where {T<:Array{Symbol}} )::Tuple where {T<:Array{String}}
_responsedictKey = keys(responsedict) _responsedictKey = keys(responsedict)
responsedictKey = [i for i in _responsedictKey] # convert into a list responsedictKey = [i for i in _responsedictKey] # convert into a list
is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys]
test/config.json
-76
View File
@@ -1,76 +0,0 @@
{
"mqttServerInfo": {
"description": "mqtt server info",
"port": 1883,
"broker": "mqtt.yiem.cc"
},
"testingOrProduction": {
"value": "testing",
"description": "agent status, couldbe testing or production"
},
"agentid": {
"value": "2b74b87a-5413-4fe2-a4d3-405891051680",
"description": "a unique id for this agent"
},
"agentCentralConfigTopic": {
"mqtttopic": "/yiem_branch_1/agent/sommelier/backend/config/api/v1.1",
"description": "a central agent server's topic to get this agent config"
},
"servicetopic": {
"mqtttopic": [
"/yiem/hq/agent/sommelier/backend/prompt/api_v1/testing"
],
"description": "a topic this agent are waiting for service request"
},
"role": {
"value": "sommelier",
"description": "agent role"
},
"organization": {
"value": "yiem_branch_1",
"description": "organization name"
},
"externalservice": {
"loadbalancer": {
"mqtttopic": "/loadbalancer/requestingservice",
"description": "text to text service with instruct LLM"
},
"text2textinstruct": {
"mqtttopic": "/loadbalancer/requestingservice",
"description": "text to text service with instruct LLM",
"llminfo": {
"name": "llama3instruct"
}
},
"virtualWineCustomer_1": {
"mqtttopic": "/virtualenvironment/winecustomer",
"description": "text to text service with instruct LLM that act as wine customer",
"llminfo": {
"name": "llama3instruct"
}
},
"text2textchat": {
"mqtttopic": "/loadbalancer/requestingservice",
"description": "text to text service with instruct LLM",
"llminfo": {
"name": "llama3instruct"
}
},
"wineDB" : {
"description": "A wine database connection info for LibPQ client",
"host": "192.168.88.12",
"port": 10201,
"dbname": "wineDB",
"user": "yiemtechnologies",
"password": "yiemtechnologies@Postgres_0.0"
},
"SQLVectorDB" : {
"description": "A wine database connection info for LibPQ client",
"host": "192.168.88.12",
"port": 10203,
"dbname": "SQLVectorDB",
"user": "yiemtechnologies",
"password": "yiemtechnologies@Postgres_0.0"
}
}
}
test/test1.jl
+291
View File
@@ -0,0 +1,291 @@
using JSON, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames, DataStructures, HTTP, Base64,
NATS, Base.Threads
using YiemAgent, GeneralUtils, msghandler
function text2text_instruct_llm(sender_id::String, openai_msg::Dict{String, Any})
payloads = [("msg", openai_msg, "dictionary")] # List of tuples
_, msg_envelope_json_str = msghandler.smartpack(
config["externalService"]["servicesloadbalancer"]["nats"],
payloads;
sender_id=sender_id,
msg_purpose="text2text",
broker_url=config["nats_server_info"]["url"],
fileserver_url=config["externalService"]["fileserver"]["url"])
reply = NATS.request(agent_conn,
config["externalService"]["servicesloadbalancer"]["nats"],
msg_envelope_json_str, timeout=120)
incoming_env_json_str = String(reply.payload)
incoming_env = msghandler.smartunpack(incoming_env_json_str)
_llm_response = incoming_env["payloads"][1][2]
llm_response = _llm_response["choices"][1]["message"]["content"]
return llm_response
end
#TESTING get text embedding from a LLM service
function get_embedding(text::AbstractArray{String})
documents_dict = Dict("documents" => text)
payloads = [("documents", documents_dict, "dictionary")]
_, msg_envelope_json_str = msghandler.smartpack(
config["externalService"]["servicesloadbalancer"]["nats"],
payloads;
msg_purpose="embedding",
broker_url=config["nats_server_info"]["url"],
fileserver_url=config["externalService"]["fileserver"]["url"])
reply = NATS.request(agent_conn,
config["externalService"]["servicesloadbalancer"]["nats"],
msg_envelope_json_str, timeout=120)
incoming_env_json_str = String(reply.payload)
incoming_env = msghandler.smartunpack(incoming_env_json_str)
embedding_response = incoming_env["payloads"][1][2]
return embedding_response
end
#TESTING
function execute_sql_winedb(config::JSON.Object, sql::T) where {T<:AbstractString}
host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
port = parse(Int, _port)
dbname = "winedb"
user = config["externalservice"]["sommpanion_db"]["user"]
password = config["externalservice"]["sommpanion_db"]["password"]
db_connection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password")
result = LibPQ.execute(db_connection, sql)
LibPQ.close(db_connection)
return result
end
#TESTING
function similar_sql_vectordb(query; maxdistance::Integer=100)
tablename = "sqlllm_decision_repository"
# get embedding of the query
df = find_similar_text_from_vectordb(query, tablename,
"function_input_embedding", execute_sql_vectordb)
# println(df[1, [:id, :function_output]])
row, col = size(df)
distance = row == 0 ? Inf : df[1, :distance]
# distance = 100 # CHANGE this is for testing only
if row != 0 && distance < maxdistance
# if there is usable SQL, return it.
output_b64 = df[1, :function_output_base64] # pick the closest match
output_str = String(base64decode(output_b64))
rowid = df[1, :id]
println("\n~~~ found similar sql. row id $rowid, distance $distance ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
return (dict=output_str, distance=distance)
else
println("\n~~~ similar sql not found, max distance $maxdistance ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
return (dict=nothing, distance=nothing)
end
end
#TESTING
function insert_sql_vectordb(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString}
tablename = "sqlllm_decision_repository"
# get embedding of the query
# query = state[:thoughtHistory][:question]
df = find_similar_text_from_vectordb(query, tablename,
"function_input_embedding", execute_sql_vectordb)
row, col = size(df)
distance = row == 0 ? Inf : df[1, :distance]
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
_query_embedding = get_embedding([query])[1]
query_embedding = _query_embedding["data"][1]["embedding"]
query = replace(query, "'" => "")
sql_base64 = base64encode(SQL)
sql_ = replace(SQL, "'" => "")
sql = """
INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$query', '$sql_', '$sql_base64', '$query_embedding');
"""
# println("\n~~~ added new decision to vectorDB ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
# println(sql)
_ = execute_sql_vectordb(sql)
end
end
#TESTING
function execute_sql_vectordb(sql::T) where {T<:AbstractString}
host_url, _port = split(config["SQLVectorDB"]["url"], ':')
port = parse(Int, _port)
dbname = config[:externalservice][:SQLVectorDB][:dbname]
user = config[:externalservice][:SQLVectorDB][:user]
password = config[:externalservice][:SQLVectorDB][:password]
DBconnection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password")
result = LibPQ.execute(DBconnection, sql)
close(DBconnection)
return result
end
function similar_sommelier_decision(recentevents::T1; maxdistance::Integer=3
)::Union{AbstractDict, Nothing} where {T1<:AbstractString}
tablename = "sommelier_decision_repository"
# find similar
println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__)
df = find_similar_text_from_vectordb(recentevents, tablename,
"function_input_embedding", execute_sql_vectordb)
row, col = size(df)
distance = row == 0 ? Inf : df[1, :distance]
if row != 0 && distance < maxdistance
# if there is usable decision, return it.
rowid = df[1, :id]
println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
output_b64 = df[1, :function_output_base64] # pick the closest match
_output_str = String(base64decode(output_b64))
output = copy(JSON.read(_output_str))
return output
else
println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
return nothing
end
end
#TESTING
function find_similar_text_from_vectordb(text::T1, tablename::T2, embeddingColumnName::T3,
vectorDB::Function; limit::Integer=1
)::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
# get embedding from LLM service
_embedding = get_embedding([text])[1]
embedding = _embedding["data"][1]["embedding"]
# check whether there is close enough vector already store in vectorDB. if no, add, else skip
sql = """
SELECT *, $embeddingColumnName <-> '$embedding' as distance
FROM $tablename
ORDER BY distance LIMIT $limit;
"""
response = vectorDB(sql)
df = DataFrame(response)
return df
end
function insert_sommelier_decision(recentevents::T1, decision::T2; maxdistance::Integer=5
) where {T1<:AbstractString, T2<:AbstractDict}
tablename = "sommelier_decision_repository"
# find similar
df = find_similar_text_from_vectordb(recentevents, tablename,
"function_input_embedding", execute_sql_vectordb)
row, col = size(df)
distance = row == 0 ? Inf : df[1, :distance]
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
_embedding = get_embedding([recentevents])[1]
recentevents_embedding = _embedding["data"][1]["embedding"]
recentevents = replace(recentevents, "'" => "")
decision_json = JSON.json(decision)
decision_base64 = base64encode(decision_json)
decision = replace(decision_json, "'" => "")
sql =
"""
INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding');
"""
println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__)
println(sql)
_ = execute_sql_vectordb(sql)
else
println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__)
end
end
sessionId = "0"
backend_session_topic = "sommpanion.backend.agentbackend.v1.inbox.$sessionId"
config = JSON.parsefile("./dummy_config.json")
agent_ch = Channel(8)
agent_conn = NATS.connect(config["nats_server_info"]["url"])
sub2 = NATS.subscribe(agent_conn, backend_session_topic) do msg
put!(agent_ch, msg)
end
agent_context = YiemAgent.agentcontext(
text2text_instruct_llm,
get_embedding,
execute_sql_winedb,
similar_sql_vectordb,
insert_sql_vectordb,
similar_sommelier_decision,
insert_sommelier_decision
)
# can't instantiate
agent = YiemAgent.sommelier(
agent_context;
name="Janie",
id=sessionId, # agent instance id
retailername="Yiem",
llmFormatName=""
)
# 1. Read local file and encode to base64 string
image1_path = "test/large_image.png"
image1_bytes = read(image1_path)
image1_base64_string = base64encode(image1_bytes)
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
mime_type = "image/png"
data1_uri = "data:$(mime_type);base64,$(image1_base64_string)"
# 3. Construct payload with the Data URI
usermsg = Dict{String, Any}(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => "รู้จักไวน์ที่อยู่ในรูปมั้ย"),
Dict(
"type" => "image_url",
"image_url" => Dict("url" => data1_uri)
)
]
)
result = YiemAgent.conversation(agent; userinput=usermsg)
println(result)