update
This commit is contained in:
@@ -1,76 +0,0 @@
|
||||
{
|
||||
"mqttServerInfo": {
|
||||
"description": "mqtt server info",
|
||||
"port": 1883,
|
||||
"broker": "mqtt.yiem.cc"
|
||||
},
|
||||
"testingOrProduction": {
|
||||
"value": "testing",
|
||||
"description": "agent status, couldbe testing or production"
|
||||
},
|
||||
"agentid": {
|
||||
"value": "2b74b87a-5413-4fe2-a4d3-405891051680",
|
||||
"description": "a unique id for this agent"
|
||||
},
|
||||
"agentCentralConfigTopic": {
|
||||
"mqtttopic": "/yiem_branch_1/agent/sommelier/backend/config/api/v1.1",
|
||||
"description": "a central agent server's topic to get this agent config"
|
||||
},
|
||||
"servicetopic": {
|
||||
"mqtttopic": [
|
||||
"/yiem/hq/agent/sommelier/backend/prompt/api_v1/testing"
|
||||
],
|
||||
"description": "a topic this agent are waiting for service request"
|
||||
},
|
||||
"role": {
|
||||
"value": "sommelier",
|
||||
"description": "agent role"
|
||||
},
|
||||
"organization": {
|
||||
"value": "yiem_branch_1",
|
||||
"description": "organization name"
|
||||
},
|
||||
"externalservice": {
|
||||
"loadbalancer": {
|
||||
"mqtttopic": "/loadbalancer/requestingservice",
|
||||
"description": "text to text service with instruct LLM"
|
||||
},
|
||||
"text2textinstruct": {
|
||||
"mqtttopic": "/loadbalancer/requestingservice",
|
||||
"description": "text to text service with instruct LLM",
|
||||
"llminfo": {
|
||||
"name": "llama3instruct"
|
||||
}
|
||||
},
|
||||
"virtualWineCustomer_1": {
|
||||
"mqtttopic": "/virtualenvironment/winecustomer",
|
||||
"description": "text to text service with instruct LLM that act as wine customer",
|
||||
"llminfo": {
|
||||
"name": "llama3instruct"
|
||||
}
|
||||
},
|
||||
"text2textchat": {
|
||||
"mqtttopic": "/loadbalancer/requestingservice",
|
||||
"description": "text to text service with instruct LLM",
|
||||
"llminfo": {
|
||||
"name": "llama3instruct"
|
||||
}
|
||||
},
|
||||
"wineDB" : {
|
||||
"description": "A wine database connection info for LibPQ client",
|
||||
"host": "192.168.88.12",
|
||||
"port": 10201,
|
||||
"dbname": "wineDB",
|
||||
"user": "yiemtechnologies",
|
||||
"password": "yiemtechnologies@Postgres_0.0"
|
||||
},
|
||||
"SQLVectorDB" : {
|
||||
"description": "A wine database connection info for LibPQ client",
|
||||
"host": "192.168.88.12",
|
||||
"port": 10203,
|
||||
"dbname": "SQLVectorDB",
|
||||
"user": "yiemtechnologies",
|
||||
"password": "yiemtechnologies@Postgres_0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
using JSON, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames, DataStructures, HTTP, Base64,
|
||||
NATS, Base.Threads
|
||||
using YiemAgent, GeneralUtils, msghandler
|
||||
|
||||
|
||||
function text2text_instruct_llm(sender_id::String, openai_msg::Dict{String, Any})
|
||||
payloads = [("msg", openai_msg, "dictionary")] # List of tuples
|
||||
_, msg_envelope_json_str = msghandler.smartpack(
|
||||
config["externalService"]["servicesloadbalancer"]["nats"],
|
||||
payloads;
|
||||
sender_id=sender_id,
|
||||
msg_purpose="text2text",
|
||||
broker_url=config["nats_server_info"]["url"],
|
||||
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||
|
||||
reply = NATS.request(agent_conn,
|
||||
config["externalService"]["servicesloadbalancer"]["nats"],
|
||||
msg_envelope_json_str, timeout=120)
|
||||
|
||||
incoming_env_json_str = String(reply.payload)
|
||||
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||
_llm_response = incoming_env["payloads"][1][2]
|
||||
llm_response = _llm_response["choices"][1]["message"]["content"]
|
||||
return llm_response
|
||||
end
|
||||
|
||||
#TESTING get text embedding from a LLM service
|
||||
function get_embedding(text::AbstractArray{String})
|
||||
documents_dict = Dict("documents" => text)
|
||||
payloads = [("documents", documents_dict, "dictionary")]
|
||||
_, msg_envelope_json_str = msghandler.smartpack(
|
||||
config["externalService"]["servicesloadbalancer"]["nats"],
|
||||
payloads;
|
||||
msg_purpose="embedding",
|
||||
broker_url=config["nats_server_info"]["url"],
|
||||
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||
|
||||
reply = NATS.request(agent_conn,
|
||||
config["externalService"]["servicesloadbalancer"]["nats"],
|
||||
msg_envelope_json_str, timeout=120)
|
||||
incoming_env_json_str = String(reply.payload)
|
||||
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||
embedding_response = incoming_env["payloads"][1][2]
|
||||
|
||||
return embedding_response
|
||||
end
|
||||
|
||||
#TESTING
|
||||
function execute_sql_winedb(config::JSON.Object, sql::T) where {T<:AbstractString}
|
||||
host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
|
||||
port = parse(Int, _port)
|
||||
dbname = "winedb"
|
||||
user = config["externalservice"]["sommpanion_db"]["user"]
|
||||
password = config["externalservice"]["sommpanion_db"]["password"]
|
||||
db_connection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password")
|
||||
result = LibPQ.execute(db_connection, sql)
|
||||
LibPQ.close(db_connection)
|
||||
return result
|
||||
end
|
||||
|
||||
#TESTING
|
||||
function similar_sql_vectordb(query; maxdistance::Integer=100)
|
||||
tablename = "sqlllm_decision_repository"
|
||||
# get embedding of the query
|
||||
df = find_similar_text_from_vectordb(query, tablename,
|
||||
"function_input_embedding", execute_sql_vectordb)
|
||||
# println(df[1, [:id, :function_output]])
|
||||
row, col = size(df)
|
||||
distance = row == 0 ? Inf : df[1, :distance]
|
||||
# distance = 100 # CHANGE this is for testing only
|
||||
if row != 0 && distance < maxdistance
|
||||
# if there is usable SQL, return it.
|
||||
output_b64 = df[1, :function_output_base64] # pick the closest match
|
||||
output_str = String(base64decode(output_b64))
|
||||
rowid = df[1, :id]
|
||||
println("\n~~~ found similar sql. row id $rowid, distance $distance ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
return (dict=output_str, distance=distance)
|
||||
else
|
||||
println("\n~~~ similar sql not found, max distance $maxdistance ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
return (dict=nothing, distance=nothing)
|
||||
end
|
||||
end
|
||||
|
||||
#TESTING
|
||||
function insert_sql_vectordb(query::T1, SQL::T2; maxdistance::Integer=3) where {T1<:AbstractString, T2<:AbstractString}
|
||||
tablename = "sqlllm_decision_repository"
|
||||
# get embedding of the query
|
||||
# query = state[:thoughtHistory][:question]
|
||||
df = find_similar_text_from_vectordb(query, tablename,
|
||||
"function_input_embedding", execute_sql_vectordb)
|
||||
row, col = size(df)
|
||||
distance = row == 0 ? Inf : df[1, :distance]
|
||||
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
|
||||
_query_embedding = get_embedding([query])[1]
|
||||
query_embedding = _query_embedding["data"][1]["embedding"]
|
||||
query = replace(query, "'" => "")
|
||||
sql_base64 = base64encode(SQL)
|
||||
sql_ = replace(SQL, "'" => "")
|
||||
|
||||
sql = """
|
||||
INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$query', '$sql_', '$sql_base64', '$query_embedding');
|
||||
"""
|
||||
# println("\n~~~ added new decision to vectorDB ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# println(sql)
|
||||
_ = execute_sql_vectordb(sql)
|
||||
end
|
||||
end
|
||||
|
||||
#TESTING
|
||||
function execute_sql_vectordb(sql::T) where {T<:AbstractString}
|
||||
host_url, _port = split(config["SQLVectorDB"]["url"], ':')
|
||||
port = parse(Int, _port)
|
||||
dbname = config[:externalservice][:SQLVectorDB][:dbname]
|
||||
user = config[:externalservice][:SQLVectorDB][:user]
|
||||
password = config[:externalservice][:SQLVectorDB][:password]
|
||||
DBconnection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password")
|
||||
result = LibPQ.execute(DBconnection, sql)
|
||||
close(DBconnection)
|
||||
return result
|
||||
end
|
||||
|
||||
|
||||
function similar_sommelier_decision(recentevents::T1; maxdistance::Integer=3
|
||||
)::Union{AbstractDict, Nothing} where {T1<:AbstractString}
|
||||
tablename = "sommelier_decision_repository"
|
||||
# find similar
|
||||
println("\n~~~ search vectorDB for this: $recentevents ", @__FILE__, " ", @__LINE__)
|
||||
df = find_similar_text_from_vectordb(recentevents, tablename,
|
||||
"function_input_embedding", execute_sql_vectordb)
|
||||
row, col = size(df)
|
||||
distance = row == 0 ? Inf : df[1, :distance]
|
||||
if row != 0 && distance < maxdistance
|
||||
# if there is usable decision, return it.
|
||||
rowid = df[1, :id]
|
||||
println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
|
||||
output_b64 = df[1, :function_output_base64] # pick the closest match
|
||||
_output_str = String(base64decode(output_b64))
|
||||
output = copy(JSON.read(_output_str))
|
||||
return output
|
||||
else
|
||||
println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
|
||||
return nothing
|
||||
end
|
||||
end
|
||||
|
||||
#TESTING
|
||||
function find_similar_text_from_vectordb(text::T1, tablename::T2, embeddingColumnName::T3,
|
||||
vectorDB::Function; limit::Integer=1
|
||||
)::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
|
||||
# get embedding from LLM service
|
||||
_embedding = get_embedding([text])[1]
|
||||
embedding = _embedding["data"][1]["embedding"]
|
||||
# check whether there is close enough vector already store in vectorDB. if no, add, else skip
|
||||
sql = """
|
||||
SELECT *, $embeddingColumnName <-> '$embedding' as distance
|
||||
FROM $tablename
|
||||
ORDER BY distance LIMIT $limit;
|
||||
"""
|
||||
response = vectorDB(sql)
|
||||
df = DataFrame(response)
|
||||
return df
|
||||
end
|
||||
|
||||
|
||||
function insert_sommelier_decision(recentevents::T1, decision::T2; maxdistance::Integer=5
|
||||
) where {T1<:AbstractString, T2<:AbstractDict}
|
||||
tablename = "sommelier_decision_repository"
|
||||
# find similar
|
||||
df = find_similar_text_from_vectordb(recentevents, tablename,
|
||||
"function_input_embedding", execute_sql_vectordb)
|
||||
row, col = size(df)
|
||||
distance = row == 0 ? Inf : df[1, :distance]
|
||||
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
|
||||
_embedding = get_embedding([recentevents])[1]
|
||||
recentevents_embedding = _embedding["data"][1]["embedding"]
|
||||
recentevents = replace(recentevents, "'" => "")
|
||||
decision_json = JSON.json(decision)
|
||||
decision_base64 = base64encode(decision_json)
|
||||
decision = replace(decision_json, "'" => "")
|
||||
|
||||
sql =
|
||||
"""
|
||||
INSERT INTO $tablename (function_input, function_output, function_output_base64, function_input_embedding) VALUES ('$recentevents', '$decision', '$decision_base64', '$recentevents_embedding');
|
||||
"""
|
||||
println("\n~~~ added new decision to vectorDB ", @__FILE__, " ", @__LINE__)
|
||||
println(sql)
|
||||
_ = execute_sql_vectordb(sql)
|
||||
else
|
||||
println("~~~ similar decision previously cached, distance $distance ", @__FILE__, " ", @__LINE__)
|
||||
end
|
||||
end
|
||||
|
||||
sessionId = "0"
|
||||
backend_session_topic = "sommpanion.backend.agentbackend.v1.inbox.$sessionId"
|
||||
|
||||
config = JSON.parsefile("./dummy_config.json")
|
||||
agent_ch = Channel(8)
|
||||
agent_conn = NATS.connect(config["nats_server_info"]["url"])
|
||||
sub2 = NATS.subscribe(agent_conn, backend_session_topic) do msg
|
||||
put!(agent_ch, msg)
|
||||
end
|
||||
|
||||
agent_context = YiemAgent.agentcontext(
|
||||
text2text_instruct_llm,
|
||||
get_embedding,
|
||||
execute_sql_winedb,
|
||||
similar_sql_vectordb,
|
||||
insert_sql_vectordb,
|
||||
similar_sommelier_decision,
|
||||
insert_sommelier_decision
|
||||
)
|
||||
|
||||
# can't instantiate
|
||||
agent = YiemAgent.sommelier(
|
||||
agent_context;
|
||||
name="Janie",
|
||||
id=sessionId, # agent instance id
|
||||
retailername="Yiem",
|
||||
llmFormatName=""
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
# 1. Read local file and encode to base64 string
|
||||
image1_path = "test/large_image.png"
|
||||
image1_bytes = read(image1_path)
|
||||
image1_base64_string = base64encode(image1_bytes)
|
||||
|
||||
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
|
||||
mime_type = "image/png"
|
||||
data1_uri = "data:$(mime_type);base64,$(image1_base64_string)"
|
||||
|
||||
# 3. Construct payload with the Data URI
|
||||
usermsg = Dict{String, Any}(
|
||||
"role" => "user",
|
||||
"content" => [
|
||||
Dict("type" => "text", "text" => "รู้จักไวน์ที่อยู่ในรูปมั้ย"),
|
||||
Dict(
|
||||
"type" => "image_url",
|
||||
"image_url" => Dict("url" => data1_uri)
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = YiemAgent.conversation(agent; userinput=usermsg)
|
||||
println(result)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user