From e63dd7d898ed0211d48c7a96517e0c7ea735d3ff Mon Sep 17 00:00:00 2001 From: narawat Date: Sat, 27 Jun 2026 08:00:41 +0700 Subject: [PATCH] update --- Manifest.toml | 55 ++++- Project.toml | 2 + dummy_config.json | 53 +++++ src/interface.jl | 549 +++++++++++----------------------------------- src/type.jl | 8 +- src/util.jl | 8 +- test/config.json | 76 ------- test/test1.jl | 291 ++++++++++++++++++++++++ 8 files changed, 534 insertions(+), 508 deletions(-) create mode 100644 dummy_config.json delete mode 100644 test/config.json create mode 100644 test/test1.jl diff --git a/Manifest.toml b/Manifest.toml index 83d63ea..2b2d363 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "7462b22f4fd62982e36c8671793df6d8908c9ad0" +project_hash = "972fb718a7c47dc6826419825a1ab5fe7b0fde84" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -14,6 +14,12 @@ version = "1.1.3" uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.2" +[[deps.Arrow]] +deps = ["ArrowTypes", "BitIntegers", "CodecLz4", "CodecZstd", "ConcurrentUtilities", "DataAPI", "Dates", "EnumX", "Mmap", "PooledArrays", "SentinelArrays", "StringViews", "Tables", "TimeZones", "TranscodingStreams", "UUIDs"] +git-tree-sha1 = "4a69a3eadc1f7da78d950d1ef270c3a62c1f7e01" +uuid = "69666777-d1a9-59fb-9406-91d4454c9d45" +version = "2.8.1" + [[deps.ArrowTypes]] deps = ["Sockets", "UUIDs"] git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101" @@ -28,6 +34,12 @@ version = "1.11.0" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" +[[deps.BitIntegers]] +deps = ["Random"] +git-tree-sha1 = "091d591a060e43df1dd35faab3ca284925c48e46" +uuid = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1" +version = "0.3.7" + [[deps.BufferedStreams]] git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643" uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" @@ -60,12 +72,24 @@ git-tree-sha1 = "40956acdbef3d8c7cc38cba42b56034af8f8581a" uuid = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf" version = "0.3.4" +[[deps.CodecLz4]] +deps = ["Lz4_jll", "TranscodingStreams"] +git-tree-sha1 = "d58afcd2833601636b48ee8cbeb2edcb086522c2" +uuid = "5ba52731-8f18-5e0d-9241-30f10d1ec561" +version = "0.4.6" + [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" version = "0.7.8" +[[deps.CodecZstd]] +deps = ["TranscodingStreams", "Zstd_jll"] +git-tree-sha1 = "da54a6cd93c54950c15adf1d336cfd7d71f51a56" +uuid = "6b39b394-51ab-5f42-8807-6242bab2b4c2" +version = "0.8.7" + [[deps.Compat]] deps = ["TOML", "UUIDs"] git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" @@ -86,6 +110,12 @@ deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" version = "1.3.0+1" +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.5.1" + [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" @@ -424,6 +454,12 @@ git-tree-sha1 = "0aad96d7b987a5600e260eec50147b254d5ff7e6" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" version = "3.6.0" +[[deps.Lz4_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "191686b1ac1ea9c89fc52e996ad15d1d241d1e33" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.1+0" + [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -662,7 +698,7 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" [[deps.SQLLLM]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "HTTP", "JSON3", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "GeneralUtils", "HTTP", "JSON3", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"] path = "../SQLLLM" uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3" version = "0.2.4" @@ -769,6 +805,11 @@ git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" version = "0.4.4" +[[deps.StringViews]] +git-tree-sha1 = "f2dcb92855b31ad92fe8f079d4f75ac57c93e4b8" +uuid = "354b36f9-a18e-4713-926e-db85100087ba" +version = "1.3.7" + [[deps.StructTypes]] deps = ["Dates", "UUIDs"] git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" @@ -884,7 +925,7 @@ uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" version = "1.6.1" [[deps.YiemAgent]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "Serialization", "URIs", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SQLLLM", "Serialization", "URIs", "UUIDs"] path = "." uuid = "e012c34b-7f78-48e0-971c-7abb83b6f0a2" version = "0.4.0" @@ -911,6 +952,14 @@ git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" version = "1.0.21+0" +[[deps.msghandler]] +deps = ["Arrow", "Base64", "DataFrames", "Dates", "GeneralUtils", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"] +git-tree-sha1 = "02825c9cb2c15d3bc30fdf28656c1f594a04cb5f" +repo-rev = "main" +repo-url = "https://git.yiem.cc/ton/msghandler" +uuid = "f2724d33-f338-4a57-b9f8-1be882570d10" +version = "0.5.6" + [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" diff --git a/Project.toml b/Project.toml index 7f390db..1a77850 100644 --- a/Project.toml +++ b/Project.toml @@ -21,6 +21,7 @@ SQLLLM = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +msghandler = "f2724d33-f338-4a57-b9f8-1be882570d10" [compat] CSV = "0.10.15" @@ -29,3 +30,4 @@ GeneralUtils = "0.4.0" HTTP = "2.4.0" JSON = "1.6.1" NATS = "0.1.0" +msghandler = "0.5.6" diff --git a/dummy_config.json b/dummy_config.json new file mode 100644 index 0000000..11d65ee --- /dev/null +++ b/dummy_config.json @@ -0,0 +1,53 @@ +{ + "nats_server_info": { + "description": "nats server", + "url": "nats.yiem.cc" + }, + "testingOrProduction": "testing", + "agentId": "2b74b87a-5413-4fe2-a4d3-405891051680", + "agentCentralConfigSubject": "/yiem/hq/agent/sommelier/backend/config/api/v1.1", + "this_service_input_channel": { + "mqtt": [ + "/yiem/hq/agent/sommpanion/backend/db/api_v1" + ], + "nats": [ + "sommpanion.backend.agentbackend.v1.inbox" + ] + }, + "agentRole": "sommelier", + "organization": "yiem_hq", + "externalService": { + "servicesloadbalancer": { + "nats": "sommpanion.backend.servicesloadbalancer.v1.inbox" + }, + "textembedding": { + "url": "textembedding.api.v1" + }, + "textimage_to_text_llm": { + "url": "https://llmcoder.yiem.cc/v1/chat/completions", + "modelname": "Qwen3.6-35B-A3B-UD-Q4_K_M" + }, + "virtualWineCustomer_1": { + "serviceSubject": "", + "modelName": "qwen3:8b" + }, + "sommpanion_db" : { + "description": "A database connection info for LibPQ client", + "url": "192.168.88.106:5432", + "dbname": "winedb", + "user": "yiemtechnologies@gmail.com", + "password": "yiemtechnologies@Postgres_0.0" + }, + "sommpanion_vectordb" : { + "description": "A wine database connection info for LibPQ client", + "url": "192.168.88.106:5433", + "dbname": "vectordb", + "user": "yiemtechnologies@gmail.com", + "password": "yiemtechnologies@Postgres_0.0" + }, + "fileserver": { + "description": "temporary file server", + "url": "https://fileserver.yiem.cc" + } + } +} \ No newline at end of file diff --git a/src/interface.jl b/src/interface.jl index 9a15e07..1ac9d5b 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -122,21 +122,21 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 # """ # end - recentevents_ind = GeneralUtils.recentElementsIndex( - length(a.memory["events"]), recentevents; includelatest=true) + # recentevents_ind = GeneralUtils.recentElementsIndex( + # length(a.memory["events"]), recentevents; includelatest=true) requiredKeys = ["plan", "actionname", "actioninput"] context = """ - $(a.memory["scratchpad"]) + $(a.memory["shortmem"]["scratchpad"]) """ - #WORKING add context to text of the latest message (in the front). + # add context to text of the latest message (in the front). # use for loop because in openai format, each msg may contain both text and image. - for d in enumerate(a.chathistory[end]["content"]) + for d in a.chathistory[end]["content"] if d["type"] == "text" d["text"] = context * d["text"] break @@ -145,28 +145,27 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 errornote = "N/A" response = nothing # placeholder for show when error msg show up - for attempt in 1:maxattempt if attempt > 1 println("\nYiemAgent decisionMaker() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - + openai_msg = Dict( "model" => "gemma-4-E4B-it-UD-Q4_K_XL", "messages" => a.chathistory, "temperature" => 0.7 ) - - response = a.context.text2textInstructLLM(prompt; senderId=a.id) - response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) + response = a.context.text2textInstructLLM(a.id, openai_msg) + # response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) response = GeneralUtils.remove_french_accents(response) think, response = GeneralUtils.extractthink(response) response = String(split(response, ", observation")[1]) # in case LLM generate observation key which it isn't supposed to response = strip(response) responsedict = nothing try - responsedict = copy(JSON.parsefile(response)) + _responsedict = JSON.parse(response) + responsedict = GeneralUtils.dictify(_responsedict, keytype=String) catch println("\nERROR YiemAgent decisionMaker() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())") continue @@ -179,21 +178,6 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") continue end - # _responsedictKey = keys(responsedict) - # responsedictKey = [i for i in _responsedictKey] # convert into a list - # is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] - - # if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys) - # errornote = "Your previous attempt has more key points than answer's required key points." - # println("\nERROR YiemAgent decisionMaker() $errornote ----(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # continue - # elseif !all(is_requiredKeys_in_responsedictKey) - # zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey) - # missingkeys = [requiredKeys[i] for i in zeroind] - # errornote = "$missingkeys are missing from your previous response" - # println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)--> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # continue - # end if responsedict["actionname"] ∉ ["CHATBOX", "CHECKWINE", "PRESENTBOX", "ENDCONVERSATION"] errornote = "Your previous attempt didn't use the given functions" @@ -204,64 +188,64 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 println("\nYiem decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") pprintln(Dict(responsedict)) - # check whether an agent recommend wines before checking inventory or recommend wines - # outside its inventory - # ask LLM whether there are any winery mentioned in the response - mentioned_winery = detectWineryName(a, response) - if mentioned_winery != "None" - mentioned_winery = String.(strip.(split(mentioned_winery, ","))) + # # check whether an agent recommend wines before checking inventory or recommend wines + # # outside its inventory + # # ask LLM whether there are any winery mentioned in the response + # mentioned_winery = detectWineryName(a, response) + # if mentioned_winery != "None" + # mentioned_winery = String.(strip.(split(mentioned_winery, ","))) - # check whether the wine is in event - isWineInEvent = false - for winename in mentioned_winery - for event in a.memory["events"] - if event["observation"] !== nothing && occursin(winename, event["observation"]) - isWineInEvent = true - break - end - end - end + # # check whether the wine is in event + # isWineInEvent = false + # for winename in mentioned_winery + # for event in a.memory["events"] + # if event["observation"] !== nothing && occursin(winename, event["observation"]) + # isWineInEvent = true + # break + # end + # end + # end - # then the agent is not supposed to recommend the wine - if isWineInEvent == false - errornote = "You recommended wines that are not in your inventory before. Please only recommend wines that you have previously found in your inventory." - println("\nERROR YiemAgent decisionMaker() $errornote $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - end + # # then the agent is not supposed to recommend the wine + # if isWineInEvent == false + # errornote = "You recommended wines that are not in your inventory before. Please only recommend wines that you have previously found in your inventory." + # println("\nERROR YiemAgent decisionMaker() $errornote $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end + # end - delete!(responsedict, :mentioned_winery) + # delete!(responsedict, :mentioned_winery) # check whether responsedict["actioninput"] is the same as previous dialogue - if !isempty(a.chathistory) && responsedict["actioninput"] == a.chathistory[end]["text"] - errornote = "In your previous attempt, you repeated the previous dialogue. Please try again." - println("\nERROR YiemAgent decisionMaker() $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end + # if !isempty(a.chathistory) && responsedict["actioninput"] == a.chathistory[end]["text"] + # errornote = "In your previous attempt, you repeated the previous dialogue. Please try again." + # println("\nERROR YiemAgent decisionMaker() $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end - evaluationdict = evaluator(a, timeline, responsedict, context) - if evaluationdict[:approval] == "no" - mentor_comment = evaluationdict[:suggestion] - errornote = "Your previous attempt was not good enough. Please try again. Here is the mentor's suggestion: $mentor_comment" - println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)--> \n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end + # evaluationdict = evaluator(a, timeline, responsedict, context) + # if evaluationdict[:approval] == "no" + # mentor_comment = evaluationdict[:suggestion] + # errornote = "Your previous attempt was not good enough. Please try again. Here is the mentor's suggestion: $mentor_comment" + # println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)--> \n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # continue + # end - # store for later training - responsedict["system"] = systemmsg - responsedict["unformatPrompt"] = unformatPrompt - responsedict["prompt"] = prompt - responsedict["context"] = context - responsedict["think"] = think - responsedict["response"] = response - # responsedict["QandA"] = QandA + # # store for later training + # responsedict["system"] = systemmsg + # responsedict["unformatPrompt"] = unformatPrompt + # responsedict["prompt"] = prompt + # responsedict["context"] = context + # responsedict["think"] = think + # responsedict["response"] = response + # # responsedict["QandA"] = QandA - # check whether there is a file path exists before writing to it - if !haskey(a.memory["shortmem"], "decisionlog") - a.memory["shortmem"]["decisionlog"] = [responsedict] - else - push!(a.memory["shortmem"]["decisionlog"], responsedict) - end + # # check whether there is a file path exists before writing to it + # if !haskey(a.memory["shortmem"], "decisionlog") + # a.memory["shortmem"]["decisionlog"] = [responsedict] + # else + # push!(a.memory["shortmem"]["decisionlog"], responsedict) + # end # # save to filename ./log/decisionlog.txt # println("\nsaving YiemAgent decisionMaker() to disk ", @__FILE__, ":", @__LINE__, " $(Dates.now())") @@ -286,7 +270,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10 # println("\nYiemAgent decisionMaker() saved to disk is done. agent $(a.id)") - responsedict["prompt"] = prompt + # responsedict["prompt"] = prompt return responsedict end error("DecisionMaker failed to generate a thought ", response) @@ -468,7 +452,7 @@ base64_string = base64encode(image_bytes) # 2. Match the MIME type according to your file extension (e.g., png, jpeg) mime_type = "image/png" -data_uri = "data:$(mime_type);base64,$(base64_string)" +data1_uri = "data:;base64," # 3. Construct payload with the Data URI message => Dict( @@ -483,33 +467,31 @@ message => Dict( ) # ---------------------------------------------- 100 --------------------------------------------- # -""" #WORKING -function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Object{String, Any}}, maximumMsg=50) +""" +function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Object{String, Any}}, + maximumMsg=50) userinput = GeneralUtils.dictify(userinput; keytype=String) + + # find text in usermsg + usertext = nothing + text_position = nothing + for (i, d) in enumerate(userinput["content"]) + if d["type"] == "text" + usertext = d["text"] + text_position = i + end + end # place holder actionname = nothing result = nothing chatresponse = nothing - if userinput === nothing - # thinking loop until AI wants to communicate with the user - chatresponse = nothing - while chatresponse === nothing - actionname, result = think(a) - if actionname ∈ ["CHATBOX", "PRESENTBOX", "ENDCONVERSATION"] - chatresponse = result - end - end - - addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg) - - return chatresponse - elseif user_text_input == "newtopic" + if usertext == "newtopic" clearhistory(a) return "Okay. What shall we talk about?" else - userinput["content"][text_position] = GeneralUtils.remove_french_accents(user_text_input) + userinput["content"][text_position]["text"] = GeneralUtils.remove_french_accents(usertext) # add usermsg to a.chathistory but how do I handle images? addNewMessage(a, "user", userinput; maximumMsg=maximumMsg) @@ -521,7 +503,11 @@ function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Obj chatresponse = result end end - addNewMessage(a, "assistant", chatresponse; maximumMsg=maximumMsg) + assistant_response = Dict{String, Any}( + "role" => "assistant", + "content" => [Dict("type" => "text", "text" => chatresponse),] + ) + addNewMessage(a, "assistant", assistant_response; maximumMsg=maximumMsg) return chatresponse end @@ -568,7 +554,6 @@ function conversation(a::Union{companion, virtualcustomer}, userinput::Dict; end """ - # Arguments # Return @@ -578,92 +563,49 @@ end julia> ``` -""" # WORKING -function think(a::T)::namedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent} +""" +function think(a::T)::NamedTuple{(:actionname, :result),Tuple{String,String}} where {T<:agent} # a.memory[:recap] = generateSituationReport(a, a.context["text"2textInstructLLM]; skiprecent=0) thoughtDict = decisionMaker(a) - actionname = thoughtDict["actionname"] - actioninput = thoughtDict["actioninput"] + # # map action and input() to llm function + # response = + # if thoughtDict["actionname"] == "CHATBOX" || thoughtDict["actionname"] == "ENDCONVERSATION" + # (result=thoughtDict["plan"], errormsg=nothing, success=true) + # elseif thoughtDict["actionname"] == "CHECKWINE" + # checkwine(a, thoughtDict["actioninput"]) + # elseif thoughtDict["actionname"] == "PRESENTBOX" + # (result=thoughtDict["actioninput"], errormsg=nothing, success=true) + # else + # error("undefined LLM function. Requesting $(thoughtDict["actionname"])") + # end - # map action and input() to llm function - response = - if actionname == "CHATBOX" || actionname == "ENDCONVERSATION" - (result=thoughtDict["plan"], errormsg=nothing, success=true) - elseif actionname == "CHECKWINE" - checkwine(a, actioninput) - elseif actionname == "PRESENTBOX" - (result=actioninput, errormsg=nothing, success=true) - # elseif actionname == "ENDCONVERSATION" - # x = "Conclude the conversation, thanks the user then goodbye and inviting them to return next time." - # (result=actioninput, errormsg=nothing, success=true) - else - error("undefined LLM function. Requesting $actionname") - end + # # this section allow LLM functions above to have different return values. + # result = haskey(response, "result") ? response["result"] : nothing + # rawresponse = haskey(response, "rawresponse") ? response["rawresponse"] : nothing + # select = haskey(response, "select") ? response["select"] : nothing + # reward::Integer = haskey(response, "reward") ? response["reward"] : 0 + # isterminal::Bool = haskey(response, "isterminal") ? response["isterminal"] : false + # errormsg::Union{AbstractString,Nothing} = haskey(response, "errormsg") ? response["errormsg"] : nothing + # success::Bool = haskey(response, "success") ? response["success"] : false - # this section allow LLM functions above to have different return values. - result = haskey(response, "result") ? response["result"] : nothing - rawresponse = haskey(response, "rawresponse") ? response["rawresponse"] : nothing - select = haskey(response, "select") ? response["select"] : nothing - reward::Integer = haskey(response, "reward") ? response["reward"] : 0 - isterminal::Bool = haskey(response, "isterminal") ? response["isterminal"] : false - errormsg::Union{AbstractString,Nothing} = haskey(response, "errormsg") ? response["errormsg"] : nothing - success::Bool = haskey(response, "success") ? response["success"] : false + result = nothing + if thoughtDict["actionname"] ∈ ["CHATBOX"] + result = thoughtDict["actioninput"] + elseif thoughtDict["actionname"] ∈ ["ENDCONVERSATION"] + # WORKING add ENDCONVERSATION guideline in to scratchpad + guideline = + """ + To end conversation with the user + """ - # # manage memory (pass msg to generatechat) - # if actionname ∈ ["CHATBOX", "PRESENTBOX", "ENDCONVERSATION"] - # chatresponse = generatechat(a, thoughtDict) - # push!(a.memory["events"], - # eventdict(; - # event_description="the assistant talks to the user.", - # timestamp=Dates.now(), - # subject="assistant", - # thought=thoughtDict, - # actionname=actionname, - # actioninput=actioninput, - # ) - # ) - # result = chatresponse - if actionname ∈ ["CHATBOX"] - push!(a.memory["events"], - eventdict(; - event_description="an assistant talks to the user.", - timestamp=Dates.now(), - subject="assistant", - thought=thoughtDict, - actionname=actionname, - actioninput=actioninput, - ) - ) - result = actioninput - elseif actionname ∈ ["ENDCONVERSATION"] - chatresponse = generatechat(a, thoughtDict) - push!(a.memory["events"], - eventdict(; - event_description="an assistant talks to the user to end conversation.", - timestamp=Dates.now(), - subject="assistant", - thought=thoughtDict, - actionname=actionname, - actioninput=chatresponse, - ) - ) + a.memory["shortmem"]["scratchpad"] = result = chatresponse - elseif actionname ∈ ["PRESENTBOX"] - chatresponse = presentbox(a, thoughtDict) - push!(a.memory["events"], - eventdict(; - event_description="the assistant presents wines to the user.", - timestamp=Dates.now(), - subject="assistant", - thought=thoughtDict, - actionname=actionname, - actioninput=chatresponse, - ) - ) + elseif thoughtDict["actionname"] ∈ ["PRESENTBOX"] + chatresponse = presentbox(a, thoughtDict) #PENDING result = chatresponse - elseif actionname == "CHECKWINE" + elseif thoughtDict["actionname"] == "CHECKWINE" if rawresponse !== nothing vd = GeneralUtils.dfToVectorDict(rawresponse) # comes in dataframe format # a.memory["shortmem"]["found_wine"] = vd # used by decisionMaker() as a short note @@ -689,24 +631,11 @@ function think(a::T)::namedTuple{(:actionname, :result),Tuple{String,String}} wh # # """ end - - push!(a.memory["events"], - eventdict(; - event_description= "the assistant searched the database.", - timestamp= Dates.now(), - subject= "assistant", - thought=thoughtDict, - actionname=actionname, - actioninput= "I search the database with this search term: $actioninput", - observation= "This is what I found:, $result" - ) - ) - else error("condition is not defined ", @__FILE__, ":", @__LINE__, " $(Dates.now())") end - return (actionname=actionname, result=result) + return (actionname=thoughtDict["actionname"], result=result) end @@ -866,94 +795,22 @@ function presentbox(a::sommelier, thoughtDict; maxtattempt::Integer=10, recentev error("presentbox() failed to generate a response") end - -""" - -# Arguments - - `a::T1` - one of ChatAgent's agent. - - `input::T2` -# Return - A JSON string of available wine - -# Example -```jldoctest -julia> -``` - -# TODO - - [] update docs - -# Signature -""" -function generatechat(a::sommelier, thoughtDict; maxattempt::Integer=10) - systemmsg = +function endconversation(a::sommelier, thoughtDict; maxattempt::Integer=10) + text = """ - Your role: - Your name is $(a.name). You are a helpful English-speaking assistant, acting as a polite, website-based sommelier for $(a.retailername)'s wine store. - Situation: - You have some thinking in mind while you are talking with the user. - Your mission: - Concentrate on your thoughts and articulate them clearly. Keep the conversation engaging. - Your responsibility does NOT includes: - - Requesting the user to place an order, make a purchase, or confirm the order. These are the job of our sales team at the store. - - Processing sales orders or engaging in any other sales-related activities. These are the job of our sales team at the store. - - Answering questions or offering additional services beyond those related to your store's wine recommendations such as discounts, quantity, rewards programs, promotions, delivery options, shipping, boxes, gift wrapping, packaging, personalized messages or something similar. These are the job of our sales team at the store. - At each round of conversation, you will be given the following: - Your ongoing conversation with the user: ... - Your thoughts: Your current thoughts in your mind - You must follow the following guidelines: - - Do not offer additional services you didn't think - You should follow the following guidelines: - - Focus on the latest conversation - - If the user interrupts, prioritize the user - - Be honest - You should then respond to the user with: - dialogue: what you want to say to the user - You should only respond in JSON format as described below: - { - "dialogue": "..." - } - Here are some examples: - Your ongoing conversation with the user: "user> hello, I need a new car\n" - Your thoughts: "I should recommend the car we have found in our inventory to the user." - {"dialogue": "We have a variety of cars available, including the Toyota Camry 2020, the Honda Civic 2021, and the Ford Mustang 2022. Which one would you like to see?"} - - Let's begin! + --- + """ - requiredKeys = [:dialogue] + requiredKeys = ["dialogue"] - # a.memory["shortmem"]["available_wine"] is a vector of dictionary - # context = - # if length(a.memory["shortmem"]["available_wine"]) != 0 - # "Wines previously found in your inventory: $(availableWineToText(a.memory["shortmem"]["available_wine"]))" - # else - # "N/A" - # end - - chathistory = chatHistoryToText(a.chathistory) - errornote = "N/A" - response = nothing # placeholder for show when error msg show up - - yourthought = thoughtDict["plan"] - # yourthought1 = nothing + system_msg = Dict( + "role" => "system", + "content" => [ + Dict("type" => "text", "text" => systemmsg), + ] + ) for attempt in 1:maxattempt - # if attempt > 1 # use to prevent LLM generate the same respond over and over - # println("\nYiemAgent generatchat() attempt $attempt/10 ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # yourthought1 = paraphrase(a.context["text"2textInstructLLM], yourthought) - # else - # yourthought1 = yourthought - # end - - context = - """ - - Your ongoing conversation with the user: $chathistory - Your thoughts: $yourthought - P.S. $errornote - - """ unformatPrompt = [ @@ -1036,156 +893,6 @@ function generatechat(a::sommelier, thoughtDict; maxattempt::Integer=10) end -function generatechat(a::companion; recentevents::Integer=10, - converPartnerName::Union{String, Nothing}=nothing, maxattempt=10) - - recentchat_ind = GeneralUtils.recentElementsIndex(length(a.chathistory), recentevents; - includelatest=true); - recentchat = createChatLog(a.chathistory; index=recentchat_ind) - - response = nothing # placeholder for show when error msg show up - errornote = "N/A" - - for attempt in 1:maxattempt - if attempt > 1 - println("\nYiemAgent generatechat() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - end - - context = - """ - - P.S. $errornote - - """ - - unformatPrompt = - [ - Dict("name" => "system", "text" => a.systemmsg), - ] - - unformatPrompt = vcat(unformatPrompt, recentchat) - # put in model format - prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) - # add info - prompt = prompt * context - - - # replace user and assistant with partner name - prompt = replace(_prompt, "|>user"=>"|>$(converPartnerName)") - prompt = replace(prompt, "|>assistant"=>"|>$(a.name)") - - response = a.context.text2textInstructLLM(prompt; llmkwargs=llmkwargs, senderId=a.id) - response = replace(response, "<|im_start|>"=> "") - response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) - think, response = GeneralUtils.extractthink(response) - - # check whether LLM just repeat the previous dialogue - for msg in a.chathistory - if msg["text"] == response - errornote = "In your previous attempt, you repeated the previous dialogue. Please try again." - println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - end - - return response - end - error("generatechat failed to generate a response") -end - -# modify it to work with customer object -function generatechat(a::virtualcustomer; - converPartnerName::Union{String, Nothing}=nothing, maxattempt=10, recentEventNum=10 - ) - - recent_ind = GeneralUtils.recentElementsIndex(length(a.memory["events"]), recentEventNum; includelatest=true) - recentEventsDict = createEventsLog(a.memory["events"]; index=recent_ind) - response = nothing # placeholder for show when error msg show up - errornote = "N/A" - header = ["Dialogue:", "Role:"] - dictkey = ["dialogue", "role"] - llmkwargs=Dict( - "num_ctx" => 32768, - "temperature" => 0.5, - ) - - for attempt in 1:maxattempt - if attempt > 1 - println("\nYiemAgent generatechat() attempt $attempt/$maxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - end - context = - """ - - P.S. $errornote - - """ - - unformatPrompt = - [ - Dict("name" => "system", "text" => a.systemmsg), - ] - unformatPrompt = vcat(unformatPrompt, recentEventsDict) - # put in model format - prompt = GeneralUtils.formatLLMtext(unformatPrompt, a.llmFormatName) - # add info - prompt = prompt * context - - response = a.context.text2textInstructLLM(prompt; llmkwargs=llmkwargs, senderId=a.id) - response = replace(response, "<|im_start|>"=> "") - response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName) - think, response = GeneralUtils.extractthink(response) - - # check whether response has all header - detected_kw = GeneralUtils.detectKeywordVariation(header, response) - missingkeys = [k for (k, v) in detected_kw if v === nothing] - - if !isempty(missingkeys) - errornote = "$missingkeys are missing from your previous response" - println("\nERROR YiemAgent rolegenerator() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - elseif sum([length(i) for i in values(detected_kw)]) > length(header) - errornote = "\nYour previous attempt has duplicated points according to the required response format" - println("\nERROR YiemAgent rolegenerator() $errornote ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - - responsedict = GeneralUtils.textToDict(response, header; - dictKey=dictkey, symbolkey=true) - if responsedict["role"] == "no" - errornote = "In your previous attempt you said $(responsedict["dialogue"]) which you, as a customer of a wine store, are not supposed to speak." - println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - - # check if the dialogue is the same as the previous one - if length(responsedict["dialogue"]) != 0 && responsedict["dialogue"] == a.chathistory[end]["text"] - errornote = "In your previous attempt you said $(responsedict["dialogue"]) which was the same as the previous one." - println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - - # check whether LLM just repeat the previous dialogue - dublicate = false - for msg in a.chathistory - if msg["text"] == responsedict["dialogue"] - errornote = "In your previous attempt, you repeated the earlier dialogue. Please try again." - println("\nYiemAgent generatechat() $errornote:\n$response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - dublicate = true - break - end - end - if dublicate - continue - end - - # println("\n$prompt", @__FILE__, ":", @__LINE__, " $(Dates.now())") - # println("\n $response") - return responsedict["dialogue"] - end - error("generatechat failed to generate a response") -end - - function generatequestion(a, text2textInstructLLM::Function, timeline)::String systemmsg = """ diff --git a/src/type.jl b/src/type.jl index 76f4de3..55f40cd 100644 --- a/src/type.jl +++ b/src/type.jl @@ -171,7 +171,7 @@ function sommelier( image1_bytes = read(image1_path) | this part must be done image1_base64_string = base64encode(image1_bytes) | in frontend mime_type = "image/png" | not in agent code - data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" --- + data1_uri = "data:;base64," --- chathistory= [ Dict( @@ -193,7 +193,7 @@ function sommelier( "image_url" => Dict("url" => data1_uri) ), ] - ) + ), ] """ memory = Dict{String, Any}( @@ -249,8 +249,8 @@ function sommelier( Example query 1: "Dry, full-bodied red wine from 1) region: Burgundy, country: France or 2) region: Tuscany, country: Italy. Grape varietal: Merlot or Syrah. price 100 to 1000 USD." Example query 2: "Red or white wine, medium tannin, price under 700 USD" Example query 3: "white wine, region: Tuscany or Bordeaux, country: Italy or France - - PRESENTBOX which you can use to present wines you have found in your inventory to the user. The input are wine names that you want to present. The output is presentation of the wines. - - ENDCONVERSATION which you can use to properly end the conversation with the user. Input is a dialogue where you wrap up the conversation, thank the user, and invite them to return next time. + - PRESENTBOX which you can use to to generate proper dialogue to present wines you have found in your inventory to the user. The input are wine names that you want to present. The output is presentation of the wines. + - ENDCONVERSATION which you can use to generates a natural dialogue to gracefully close the conversation with the user. The input is "nothing" keyword. The output is a natural, context-aware dialogue extension that smoothly concludes the conversation. Your name is $(newAgent.name). You are a sommelier for website-based $(newAgent.retailername)'s wine store. You are working under your mentor supervision. diff --git a/src/util.jl b/src/util.jl index 353f637..1186d1e 100644 --- a/src/util.jl +++ b/src/util.jl @@ -35,7 +35,7 @@ julia> agentConfig = Dict( "text2text"=>Dict( "mqtttopic"=> "testtopic/text2text", ), - ) + ) julia> a = YiemAgent.sommelier( client, msgMeta, @@ -301,7 +301,7 @@ function createTimeline(events::T1; eventindex::Union{UnitRange, Nothing}=nothin timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n" else timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"])\\n" - end + end end # Return formatted timeline string @@ -328,7 +328,7 @@ end # timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: Not done yet.\n" # # If outcome exists, include it in formatting # else - # timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n" + # timeline *= "Event_$i $(event["subject"])> actionname: $(event["actionname"]), actioninput: $(event["actioninput"]), observation: $(event["observation"])\\n" # end # end @@ -422,7 +422,7 @@ end function checkAgentResponse_JSON(responsedict::Dict, requiredKeys::T - )::Tuple where {T<:Array{Symbol}} + )::Tuple where {T<:Array{String}} _responsedictKey = keys(responsedict) responsedictKey = [i for i in _responsedictKey] # convert into a list is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys] diff --git a/test/config.json b/test/config.json deleted file mode 100644 index 13ca00f..0000000 --- a/test/config.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "mqttServerInfo": { - "description": "mqtt server info", - "port": 1883, - "broker": "mqtt.yiem.cc" - }, - "testingOrProduction": { - "value": "testing", - "description": "agent status, couldbe testing or production" - }, - "agentid": { - "value": "2b74b87a-5413-4fe2-a4d3-405891051680", - "description": "a unique id for this agent" - }, - "agentCentralConfigTopic": { - "mqtttopic": "/yiem_branch_1/agent/sommelier/backend/config/api/v1.1", - "description": "a central agent server's topic to get this agent config" - }, - "servicetopic": { - "mqtttopic": [ - "/yiem/hq/agent/sommelier/backend/prompt/api_v1/testing" - ], - "description": "a topic this agent are waiting for service request" - }, - "role": { - "value": "sommelier", - "description": "agent role" - }, - "organization": { - "value": "yiem_branch_1", - "description": "organization name" - }, - "externalservice": { - "loadbalancer": { - "mqtttopic": "/loadbalancer/requestingservice", - "description": "text to text service with instruct LLM" - }, - "text2textinstruct": { - "mqtttopic": "/loadbalancer/requestingservice", - "description": "text to text service with instruct LLM", - "llminfo": { - "name": "llama3instruct" - } - }, - "virtualWineCustomer_1": { - "mqtttopic": "/virtualenvironment/winecustomer", - "description": "text to text service with instruct LLM that act as wine customer", - "llminfo": { - "name": "llama3instruct" - } - }, - "text2textchat": { - "mqtttopic": "/loadbalancer/requestingservice", - "description": "text to text service with instruct LLM", - "llminfo": { - "name": "llama3instruct" - } - }, - "wineDB" : { - "description": "A wine database connection info for LibPQ client", - "host": "192.168.88.12", - "port": 10201, - "dbname": "wineDB", - "user": "yiemtechnologies", - "password": "yiemtechnologies@Postgres_0.0" - }, - "SQLVectorDB" : { - "description": "A wine database connection info for LibPQ client", - "host": "192.168.88.12", - "port": 10203, - "dbname": "SQLVectorDB", - "user": "yiemtechnologies", - "password": "yiemtechnologies@Postgres_0.0" - } - } -} \ No newline at end of file diff --git a/test/test1.jl b/test/test1.jl new file mode 100644 index 0000000..654c4dc --- /dev/null +++ b/test/test1.jl @@ -0,0 +1,291 @@ +using JSON, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames, DataStructures, HTTP, Base64, + NATS, Base.Threads +using YiemAgent, GeneralUtils, msghandler + + +function text2text_instruct_llm(sender_id::String, openai_msg::Dict{String, Any}) + payloads = [("msg", openai_msg, "dictionary")] # List of tuples + _, msg_envelope_json_str = msghandler.smartpack( + config["externalService"]["servicesloadbalancer"]["nats"], + payloads; + sender_id=sender_id, + msg_purpose="text2text", + broker_url=config["nats_server_info"]["url"], + fileserver_url=config["externalService"]["fileserver"]["url"]) + + reply = NATS.request(agent_conn, + config["externalService"]["servicesloadbalancer"]["nats"], + msg_envelope_json_str, timeout=120) + + incoming_env_json_str = String(reply.payload) + incoming_env = msghandler.smartunpack(incoming_env_json_str) + _llm_response = incoming_env["payloads"][1][2] + llm_response = _llm_response["choices"][1]["message"]["content"] + return llm_response +end + +#TESTING get text embedding from a LLM service +function get_embedding(text::AbstractArray{String}) + documents_dict = Dict("documents" => text) + payloads = [("documents", documents_dict, "dictionary")] + _, msg_envelope_json_str = msghandler.smartpack( + config["externalService"]["servicesloadbalancer"]["nats"], + payloads; + msg_purpose="embedding", + broker_url=config["nats_server_info"]["url"], + fileserver_url=config["externalService"]["fileserver"]["url"]) + + reply = NATS.request(agent_conn, + config["externalService"]["servicesloadbalancer"]["nats"], + msg_envelope_json_str, timeout=120) + incoming_env_json_str = String(reply.payload) + incoming_env = msghandler.smartunpack(incoming_env_json_str) + embedding_response = incoming_env["payloads"][1][2] + + return embedding_response +end + +#TESTING +function execute_sql_winedb(config::JSON.Object, sql::T) where {T<:AbstractString} + host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':') + port = parse(Int, _port) + dbname = "winedb" + user = config["externalservice"]["sommpanion_db"]["user"] + password = config["externalservice"]["sommpanion_db"]["password"] + db_connection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password") + result = LibPQ.execute(db_connection, sql) + LibPQ.close(db_connection) + return result +end + +#TESTING +function similar_sql_vectordb(query; maxdistance::Integer=100) + tablename = "sqlllm_decision_repository" + # get embedding of the query + df = find_similar_text_from_vectordb(query, tablename, + "function_input_embedding", execute_sql_vectordb) + # println(df[1, [:id, :function_output]]) + row, col = size(df) + distance = row == 0 ? Inf : df[1, :distance] + # distance = 100 # CHANGE this is for testing only + if row != 0 && distance < maxdistance + # if there is usable SQL, return it. + output_b64 = df[1, :function_output_base64] # pick the closest match + output_str = String(base64decode(output_b64)) + rowid = df[1, :id] + println("\n~~~ found similar sql. row id $rowid, distance $distance ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + return (dict=output_str, distance=distance) + else + println("\n~~~ similar sql not found, max distance $maxdistance ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + return (dict=nothing, distance=nothing) + end +end + +#TESTING +function insert_sql_vectordb(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString} + tablename = "sqlllm_decision_repository" + # get embedding of the query + # query = state[:thoughtHistory][:question] + df = find_similar_text_from_vectordb(query, tablename, + "function_input_embedding", execute_sql_vectordb) + row, col = size(df) + distance = row == 0 ? Inf : df[1, :distance] + if row == 0 || distance > maxdistance # no close enough SQL stored in the database + _query_embedding = get_embedding([query])[1] + query_embedding = _query_embedding["data"][1]["embedding"] + query = replace(query, "'" => "") + sql_base64 = base64encode(SQL) + sql_ = replace(SQL, "'" => "") + + sql = """ + INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$query', '$sql_', '$sql_base64', '$query_embedding'); + """ + # println("\n~~~ added new decision to vectorDB ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + # println(sql) + _ = execute_sql_vectordb(sql) + end +end + +#TESTING +function execute_sql_vectordb(sql::T) where {T<:AbstractString} + host_url, _port = split(config["SQLVectorDB"]["url"], ':') + port = parse(Int, _port) + dbname = config[:externalservice][:SQLVectorDB][:dbname] + user = config[:externalservice][:SQLVectorDB][:user] + password = config[:externalservice][:SQLVectorDB][:password] + DBconnection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password") + result = LibPQ.execute(DBconnection, sql) + close(DBconnection) + return result +end + + +function similar_sommelier_decision(recentevents::T1; maxdistance::Integer=3 + )::Union{AbstractDict, Nothing} where {T1<:AbstractString} + tablename = "sommelier_decision_repository" + # find similar + println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__) + df = find_similar_text_from_vectordb(recentevents, tablename, + "function_input_embedding", execute_sql_vectordb) + row, col = size(df) + distance = row == 0 ? Inf : df[1, :distance] + if row != 0 && distance < maxdistance + # if there is usable decision, return it. + rowid = df[1, :id] + println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__) + output_b64 = df[1, :function_output_base64] # pick the closest match + _output_str = String(base64decode(output_b64)) + output = copy(JSON.read(_output_str)) + return output + else + println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__) + return nothing + end +end + +#TESTING +function find_similar_text_from_vectordb(text::T1, tablename::T2, embeddingColumnName::T3, + vectorDB::Function; limit::Integer=1 + )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString} + # get embedding from LLM service + _embedding = get_embedding([text])[1] + embedding = _embedding["data"][1]["embedding"] + # check whether there is close enough vector already store in vectorDB. if no, add, else skip + sql = """ + SELECT *, $embeddingColumnName <-> '$embedding' as distance + FROM $tablename + ORDER BY distance LIMIT $limit; + """ + response = vectorDB(sql) + df = DataFrame(response) + return df +end + + +function insert_sommelier_decision(recentevents::T1, decision::T2; maxdistance::Integer=5 + ) where {T1<:AbstractString, T2<:AbstractDict} + tablename = "sommelier_decision_repository" + # find similar + df = find_similar_text_from_vectordb(recentevents, tablename, + "function_input_embedding", execute_sql_vectordb) + row, col = size(df) + distance = row == 0 ? Inf : df[1, :distance] + if row == 0 || distance > maxdistance # no close enough SQL stored in the database + _embedding = get_embedding([recentevents])[1] + recentevents_embedding = _embedding["data"][1]["embedding"] + recentevents = replace(recentevents, "'" => "") + decision_json = JSON.json(decision) + decision_base64 = base64encode(decision_json) + decision = replace(decision_json, "'" => "") + + sql = + """ + INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding'); + """ + println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__) + println(sql) + _ = execute_sql_vectordb(sql) + else + println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__) + end +end + +sessionId = "0" +backend_session_topic = "sommpanion.backend.agentbackend.v1.inbox.$sessionId" + +config = JSON.parsefile("./dummy_config.json") +agent_ch = Channel(8) +agent_conn = NATS.connect(config["nats_server_info"]["url"]) +sub2 = NATS.subscribe(agent_conn, backend_session_topic) do msg + put!(agent_ch, msg) +end + +agent_context = YiemAgent.agentcontext( + text2text_instruct_llm, + get_embedding, + execute_sql_winedb, + similar_sql_vectordb, + insert_sql_vectordb, + similar_sommelier_decision, + insert_sommelier_decision + ) + +# can't instantiate +agent = YiemAgent.sommelier( + agent_context; + name="Janie", + id=sessionId, # agent instance id + retailername="Yiem", + llmFormatName="" +) + + + + +# 1. Read local file and encode to base64 string +image1_path = "test/large_image.png" +image1_bytes = read(image1_path) +image1_base64_string = base64encode(image1_bytes) + +# 2. Match the MIME type according to your file extension (e.g., png, jpeg) +mime_type = "image/png" +data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" + +# 3. Construct payload with the Data URI +usermsg = Dict{String, Any}( + "role" => "user", + "content" => [ + Dict("type" => "text", "text" => "รู้จักไวน์ที่อยู่ในรูปมั้ย"), + Dict( + "type" => "image_url", + "image_url" => Dict("url" => data1_uri) + ) + ] + ) + +result = YiemAgent.conversation(agent; userinput=usermsg) +println(result) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +