292 lines
9.5 KiB
Julia
292 lines
9.5 KiB
Julia
using JSON, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames, DataStructures, HTTP, Base64,
|
|
NATS, Base.Threads
|
|
using YiemAgent, GeneralUtils, msghandler
|
|
|
|
|
|
function text2text_instruct_llm(sender_id::String, openai_msg::Dict{String, Any})
|
|
payloads = [("msg", openai_msg, "dictionary")] # List of tuples
|
|
_, msg_envelope_json_str = msghandler.smartpack(
|
|
config["externalService"]["servicesloadbalancer"]["nats"],
|
|
payloads;
|
|
sender_id=sender_id,
|
|
msg_purpose="text2text",
|
|
broker_url=config["nats_server_info"]["url"],
|
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
|
|
|
reply = NATS.request(agent_conn,
|
|
config["externalService"]["servicesloadbalancer"]["nats"],
|
|
msg_envelope_json_str, timeout=120)
|
|
|
|
incoming_env_json_str = String(reply.payload)
|
|
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
|
_llm_response = incoming_env["payloads"][1][2]
|
|
llm_response = _llm_response["choices"][1]["message"]["content"]
|
|
return llm_response
|
|
end
|
|
|
|
#TESTING get text embedding from a LLM service
|
|
function get_embedding(text::AbstractArray{String})
|
|
documents_dict = Dict("documents" => text)
|
|
payloads = [("documents", documents_dict, "dictionary")]
|
|
_, msg_envelope_json_str = msghandler.smartpack(
|
|
config["externalService"]["servicesloadbalancer"]["nats"],
|
|
payloads;
|
|
msg_purpose="embedding",
|
|
broker_url=config["nats_server_info"]["url"],
|
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
|
|
|
reply = NATS.request(agent_conn,
|
|
config["externalService"]["servicesloadbalancer"]["nats"],
|
|
msg_envelope_json_str, timeout=120)
|
|
incoming_env_json_str = String(reply.payload)
|
|
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
|
embedding_response = incoming_env["payloads"][1][2]
|
|
|
|
return embedding_response
|
|
end
|
|
|
|
#TESTING
|
|
function execute_sql_winedb(config::JSON.Object, sql::T) where {T<:AbstractString}
|
|
host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
|
|
port = parse(Int, _port)
|
|
dbname = "winedb"
|
|
user = config["externalservice"]["sommpanion_db"]["user"]
|
|
password = config["externalservice"]["sommpanion_db"]["password"]
|
|
db_connection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password")
|
|
result = LibPQ.execute(db_connection, sql)
|
|
LibPQ.close(db_connection)
|
|
return result
|
|
end
|
|
|
|
#TESTING
|
|
function similar_sql_vectordb(query; maxdistance::Integer=100)
|
|
tablename = "sqlllm_decision_repository"
|
|
# get embedding of the query
|
|
df = find_similar_text_from_vectordb(query, tablename,
|
|
"function_input_embedding", execute_sql_vectordb)
|
|
# println(df[1, [:id, :function_output]])
|
|
row, col = size(df)
|
|
distance = row == 0 ? Inf : df[1, :distance]
|
|
# distance = 100 # CHANGE this is for testing only
|
|
if row != 0 && distance < maxdistance
|
|
# if there is usable SQL, return it.
|
|
output_b64 = df[1, :function_output_base64] # pick the closest match
|
|
output_str = String(base64decode(output_b64))
|
|
rowid = df[1, :id]
|
|
println("\n~~~ found similar sql. row id $rowid, distance $distance ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
|
return (dict=output_str, distance=distance)
|
|
else
|
|
println("\n~~~ similar sql not found, max distance $maxdistance ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
|
return (dict=nothing, distance=nothing)
|
|
end
|
|
end
|
|
|
|
#TESTING
|
|
function insert_sql_vectordb(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString}
|
|
tablename = "sqlllm_decision_repository"
|
|
# get embedding of the query
|
|
# query = state[:thoughtHistory][:question]
|
|
df = find_similar_text_from_vectordb(query, tablename,
|
|
"function_input_embedding", execute_sql_vectordb)
|
|
row, col = size(df)
|
|
distance = row == 0 ? Inf : df[1, :distance]
|
|
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
|
|
_query_embedding = get_embedding([query])[1]
|
|
query_embedding = _query_embedding["data"][1]["embedding"]
|
|
query = replace(query, "'" => "")
|
|
sql_base64 = base64encode(SQL)
|
|
sql_ = replace(SQL, "'" => "")
|
|
|
|
sql = """
|
|
INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$query', '$sql_', '$sql_base64', '$query_embedding');
|
|
"""
|
|
# println("\n~~~ added new decision to vectorDB ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
|
# println(sql)
|
|
_ = execute_sql_vectordb(sql)
|
|
end
|
|
end
|
|
|
|
#TESTING
|
|
function execute_sql_vectordb(sql::T) where {T<:AbstractString}
|
|
host_url, _port = split(config["SQLVectorDB"]["url"], ':')
|
|
port = parse(Int, _port)
|
|
dbname = config[:externalservice][:SQLVectorDB][:dbname]
|
|
user = config[:externalservice][:SQLVectorDB][:user]
|
|
password = config[:externalservice][:SQLVectorDB][:password]
|
|
DBconnection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password")
|
|
result = LibPQ.execute(DBconnection, sql)
|
|
close(DBconnection)
|
|
return result
|
|
end
|
|
|
|
|
|
function similar_sommelier_decision(recentevents::T1; maxdistance::Integer=3
|
|
)::Union{AbstractDict, Nothing} where {T1<:AbstractString}
|
|
tablename = "sommelier_decision_repository"
|
|
# find similar
|
|
println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__)
|
|
df = find_similar_text_from_vectordb(recentevents, tablename,
|
|
"function_input_embedding", execute_sql_vectordb)
|
|
row, col = size(df)
|
|
distance = row == 0 ? Inf : df[1, :distance]
|
|
if row != 0 && distance < maxdistance
|
|
# if there is usable decision, return it.
|
|
rowid = df[1, :id]
|
|
println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
|
|
output_b64 = df[1, :function_output_base64] # pick the closest match
|
|
_output_str = String(base64decode(output_b64))
|
|
output = copy(JSON.read(_output_str))
|
|
return output
|
|
else
|
|
println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
|
|
return nothing
|
|
end
|
|
end
|
|
|
|
#TESTING
|
|
function find_similar_text_from_vectordb(text::T1, tablename::T2, embeddingColumnName::T3,
|
|
vectorDB::Function; limit::Integer=1
|
|
)::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
|
|
# get embedding from LLM service
|
|
_embedding = get_embedding([text])[1]
|
|
embedding = _embedding["data"][1]["embedding"]
|
|
# check whether there is close enough vector already store in vectorDB. if no, add, else skip
|
|
sql = """
|
|
SELECT *, $embeddingColumnName <-> '$embedding' as distance
|
|
FROM $tablename
|
|
ORDER BY distance LIMIT $limit;
|
|
"""
|
|
response = vectorDB(sql)
|
|
df = DataFrame(response)
|
|
return df
|
|
end
|
|
|
|
|
|
function insert_sommelier_decision(recentevents::T1, decision::T2; maxdistance::Integer=5
|
|
) where {T1<:AbstractString, T2<:AbstractDict}
|
|
tablename = "sommelier_decision_repository"
|
|
# find similar
|
|
df = find_similar_text_from_vectordb(recentevents, tablename,
|
|
"function_input_embedding", execute_sql_vectordb)
|
|
row, col = size(df)
|
|
distance = row == 0 ? Inf : df[1, :distance]
|
|
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
|
|
_embedding = get_embedding([recentevents])[1]
|
|
recentevents_embedding = _embedding["data"][1]["embedding"]
|
|
recentevents = replace(recentevents, "'" => "")
|
|
decision_json = JSON.json(decision)
|
|
decision_base64 = base64encode(decision_json)
|
|
decision = replace(decision_json, "'" => "")
|
|
|
|
sql =
|
|
"""
|
|
INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding');
|
|
"""
|
|
println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__)
|
|
println(sql)
|
|
_ = execute_sql_vectordb(sql)
|
|
else
|
|
println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__)
|
|
end
|
|
end
|
|
|
|
sessionId = "0"
|
|
backend_session_topic = "sommpanion.backend.agentbackend.v1.inbox.$sessionId"
|
|
|
|
config = JSON.parsefile("./dummy_config.json")
|
|
agent_ch = Channel(8)
|
|
agent_conn = NATS.connect(config["nats_server_info"]["url"])
|
|
sub2 = NATS.subscribe(agent_conn, backend_session_topic) do msg
|
|
put!(agent_ch, msg)
|
|
end
|
|
|
|
agent_context = YiemAgent.agentcontext(
|
|
text2text_instruct_llm,
|
|
get_embedding,
|
|
execute_sql_winedb,
|
|
similar_sql_vectordb,
|
|
insert_sql_vectordb,
|
|
similar_sommelier_decision,
|
|
insert_sommelier_decision
|
|
)
|
|
|
|
# can't instantiate
|
|
agent = YiemAgent.sommelier(
|
|
agent_context;
|
|
name="Janie",
|
|
id=sessionId, # agent instance id
|
|
retailername="Yiem",
|
|
llmFormatName=""
|
|
)
|
|
|
|
|
|
|
|
|
|
# 1. Read local file and encode to base64 string
|
|
image1_path = "test/large_image.png"
|
|
image1_bytes = read(image1_path)
|
|
image1_base64_string = base64encode(image1_bytes)
|
|
|
|
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
|
|
mime_type = "image/png"
|
|
data1_uri = "data:$(mime_type);base64,$(image1_base64_string)"
|
|
|
|
# 3. Construct payload with the Data URI
|
|
usermsg = Dict{String, Any}(
|
|
"role" => "user",
|
|
"content" => [
|
|
Dict("type" => "text", "text" => "รู้จักไวน์ที่อยู่ในรูปมั้ย"),
|
|
Dict(
|
|
"type" => "image_url",
|
|
"image_url" => Dict("url" => data1_uri)
|
|
)
|
|
]
|
|
)
|
|
|
|
result = YiemAgent.conversation(agent; userinput=usermsg)
|
|
println(result)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|