using JSON, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames, DataStructures, HTTP, Base64, NATS, Base.Threads using YiemAgent, GeneralUtils, msghandler function text2text_instruct_llm(sender_id::String, openai_msg::Dict{String, Any}) payloads = [("msg", openai_msg, "dictionary")] # List of tuples _, msg_envelope_json_str = msghandler.smartpack( config["externalservice"]["servicesloadbalancer"]["nats"], payloads; sender_id=sender_id, msg_purpose="text2text", broker_url=config["nats_server_info"]["url"], fileserver_url=config["externalservice"]["fileserver"]["url"]) reply = NATS.request(agent_conn, config["externalservice"]["servicesloadbalancer"]["nats"], msg_envelope_json_str, timeout=120) incoming_env_json_str = String(reply.payload) incoming_env = msghandler.smartunpack(incoming_env_json_str) _llm_response = incoming_env["payloads"][1][2] llm_response = _llm_response["choices"][1]["message"]["content"] return llm_response end """ get a single text embedding from a LLM service Example text = ["hello"] embedding = get_embedding(text) """ function get_embedding(text::AbstractArray{String}) documents_dict = Dict("documents" => text) payloads = [("documents", documents_dict, "dictionary")] _, msg_envelope_json_str = msghandler.smartpack( config["externalservice"]["servicesloadbalancer"]["nats"], payloads; msg_purpose="embedding", broker_url=config["nats_server_info"]["url"], fileserver_url=config["externalservice"]["fileserver"]["url"]) reply = NATS.request(agent_conn, config["externalservice"]["servicesloadbalancer"]["nats"], msg_envelope_json_str, timeout=120) incoming_env_json_str = String(reply.payload) incoming_env = msghandler.smartunpack(incoming_env_json_str) embedding_response = incoming_env["payloads"][1][2] return embedding_response end """ sql = "SELECT * FROM wine;" result = execute_sql_winedb(sql) """ function execute_sql_winedb(sql::T) where {T<:AbstractString} host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':') port = parse(Int, _port) dbname = "winedb" user = config["externalservice"]["sommpanion_db"]["user"] password = config["externalservice"]["sommpanion_db"]["password"] db_connection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password") result = nothing try result = LibPQ.execute(db_connection, sql) catch e LibPQ.close(db_connection) end LibPQ.close(db_connection) return result end """ find similar sql from vector database sql = "SELECT * FROM wine;" result, distance = similar_sql_vectordb(sql) """ function similar_sql_vectordb(sql::T; maxdistance::Number=0.2) where {T<:AbstractString} tablename = "sqlllm_decision_repository" # get embedding of the query df = find_similar_text_from_vectordb(sql, tablename, "function_input_embedding", execute_sql_vectordb) # println(df[1, [:id, :function_output]]) row, col = size(df) distance = row == 0 ? Inf : df[1, :distance] if row != 0 && distance < maxdistance # if there is usable SQL, return it. output_b64 = df[1, :function_output_base64] # pick the closest match output_str = String(base64decode(output_b64)) rowid = df[1, :id] println("\n--| similar sql found. row id $rowid, distance $distance ", @__FILE__, ":", @__LINE__, " $(Dates.now())") pprintln(output_str) return (result=output_str, distance=distance) else println("\n--| similar sql not found, max distance $maxdistance ", @__FILE__, ":", @__LINE__, " $(Dates.now())") return (result=nothing, distance=nothing) end end """ insert query and sql into vector database query = "get all wines from wine table" sql = "SELECT * FROM wine;" insert_sql_vectordb(query, sql) """ function insert_sql_vectordb(query::T1, SQL::T2; maxdistance::Number=3 ) where {T1<:AbstractString, T2<:AbstractString} tablename = "sqlllm_decision_repository" # get embedding of the query # query = state[:thoughtHistory][:question] df = find_similar_text_from_vectordb(query, tablename, "function_input_embedding", execute_sql_vectordb) row, col = size(df) distance = row == 0 ? Inf : df[1, :distance] if row == 0 || distance > maxdistance # no close enough SQL stored in the database _query_embedding = get_embedding([query]) _query_embedding = GeneralUtils.dictify(_query_embedding) # println("\n--- _query_embedding() ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println(_query_embedding) # println("---\n") query_embedding = _query_embedding["data"][1]["embedding"] query = replace(query, "'" => "") sql_base64 = base64encode(SQL) sql_ = replace(SQL, "'" => "") sql = """ INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$query', '$sql_', '$sql_base64', '$query_embedding'); """ # println("\n--| added new decision to vectorDB ", @__FILE__, ":", @__LINE__, " $(Dates.now())") # println(sql) _ = execute_sql_vectordb(sql) end end """ execute sql against vectordb sql = "SELECT * FROM wine;" result = execute_sql_vectordb(sql) """ function execute_sql_vectordb(sql::T) where {T<:AbstractString} host_url, _port = split(config["externalservice"]["sommpanion_vectordb"]["url"], ':') port = parse(Int, _port) dbname = config["externalservice"]["sommpanion_vectordb"]["dbname"] user = config["externalservice"]["sommpanion_vectordb"]["user"] password = config["externalservice"]["sommpanion_vectordb"]["password"] DBconnection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password") result = LibPQ.execute(DBconnection, sql) close(DBconnection) return result end """ search similar decision llm made from vectordb """ function similar_sommelier_decision(recentevents::T1; maxdistance::Integer=3 )::Union{AbstractDict, Nothing} where {T1<:AbstractString} tablename = "sommelier_decision_repository" # find similar df = find_similar_text_from_vectordb(recentevents, tablename, "function_input_embedding", execute_sql_vectordb) row, col = size(df) distance = row == 0 ? Inf : df[1, :distance] if row != 0 && distance < maxdistance # if there is usable decision, return it. rowid = df[1, :id] println("\n--| found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__) output_b64 = df[1, :function_output_base64] # pick the closest match _output_str = String(base64decode(output_b64)) output = copy(JSON.read(_output_str)) return output else println("\n--| similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__) return nothing end end """ search similar text from vectordb """ function find_similar_text_from_vectordb(text::T1, tablename::T2, embeddingColumnName::T3, vectorDB::Function; limit::Integer=1 )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString} # get embedding from LLM service _embedding = get_embedding([text]) _embedding = _embedding["data"][1]["embedding"] _embedding = "$_embedding" embedding = _embedding[4:end] # check whether there is close enough vector already store in vectorDB. if no, add, else skip sql = """ SELECT *, $embeddingColumnName <-> '$embedding' as distance FROM $tablename ORDER BY distance LIMIT $limit; """ response = vectorDB(sql) df = DataFrame(response) return df end """ insert decision llm made to vectordb """ function insert_sommelier_decision(recentevents::T1, decision::T2; maxdistance::Integer=5 ) where {T1<:AbstractString, T2<:AbstractDict} tablename = "sommelier_decision_repository" # find similar df = find_similar_text_from_vectordb(recentevents, tablename, "function_input_embedding", execute_sql_vectordb) row, col = size(df) distance = row == 0 ? Inf : df[1, :distance] if row == 0 || distance > maxdistance # no close enough SQL stored in the database _embedding = get_embedding([recentevents])[1] recentevents_embedding = _embedding["data"][1]["embedding"] recentevents = replace(recentevents, "'" => "") decision_json = JSON.json(decision) decision_base64 = base64encode(decision_json) decision = replace(decision_json, "'" => "") sql = """ INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding'); """ println("\n--| added new decision to vectorDB ", @__FILE__, " ", @__LINE__) println(sql) _ = execute_sql_vectordb(sql) else println("--| similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__) end end config = JSON.parsefile("./appconfig.json") sessionId = "0" backend_session_topic = "sommpanion.testsubject" agent_ch = Channel(8) agent_conn = NATS.connect(config["nats_server_info"]["url"]) sub2 = NATS.subscribe(agent_conn, backend_session_topic) do msg put!(agent_ch, msg) end agent_context = YiemAgent.agentcontext( text2text_instruct_llm, get_embedding, execute_sql_winedb, similar_sql_vectordb, insert_sql_vectordb, similar_sommelier_decision, insert_sommelier_decision ) # can't instantiate agent = YiemAgent.sommelier( agent_context; name="Janie", id=sessionId, # agent instance id retailername="Yiem Wine Ltd.", llmFormatName="" ) image1_path = "test/large_image.png" image1_bytes = read(image1_path) image1_base64_string = base64encode(image1_bytes) mime_type = "image/png" data1_uri = "data:$(mime_type);base64,$(image1_base64_string)" # 1. Read local file and encode to base64 string image2_path = "test/small_image.png" image2_bytes = read(image2_path) image2_base64_string = base64encode(image2_bytes) mime_type = "image/png" data2_uri = "data:$(mime_type);base64,$(image2_base64_string)" # 3. Construct payload with the Data URI message = Dict( "role" => "user", "content" => [ Dict("type" => "text", "text" => "Do you know type of wine in the image?"), Dict( "type" => "image_url", "image_url" => Dict("url" => data1_uri) ) ] ) result = YiemAgent.conversation(agent; userinput=message) println("\n$result") # message = Dict( # "role" => "user", # "content" => [ # Dict("type" => "text", "text" => # " # เป็นงานเลี้ยงทั่วไป # "), # ] # ) # result = YiemAgent.conversation(agent; userinput=message) # println("\n$result") # message = Dict( # "role" => "user", # "content" => [ # Dict("type" => "text", "text" => "no thanks. that's all"), # ] # ) # result = YiemAgent.conversation(agent; userinput=message) # println("\n$result") # message = Dict( # "role" => "user", # "content" => [ # Dict("type" => "text", "text" => "What about this wine?"), # Dict( # "type" => "image_url", # "image_url" => Dict("url" => data2_uri) # ) # ] # ) # result = YiemAgent.conversation(agent; userinput=message) # println("\n$result")