Compare commits

...

14 Commits

Author SHA1 Message Date
ton a3ab288cfe update 2026-07-22 10:18:28 +07:00
ton 17b0974d82 Merge pull request 'v0.7.4-predefine_wine_search' (#30) from v0.7.4-predefine_wine_search into v0.7.4
Reviewed-on: #30
2026-07-21 13:17:25 +00:00
ton 8fed0b5e8f update 2026-07-21 20:11:12 +07:00
ton 567d6b79d7 up version 2026-07-21 18:02:02 +07:00
ton 74be3e5717 Merge pull request 'v0.7.3' (#29) from v0.7.3 into main
Reviewed-on: #29
2026-07-20 05:26:19 +00:00
ton 402b6fcadd Merge pull request 'update' (#28) from v0.7.3-fix_item_info into v0.7.3
Reviewed-on: #28
2026-07-20 05:24:33 +00:00
ton ef523aaa48 update 2026-07-20 12:23:03 +07:00
ton ddbb135b6b update 2026-07-17 12:22:56 +07:00
ton afda364484 update 2026-07-17 12:03:36 +07:00
ton af73d955eb Merge pull request 'v0.7.1' (#27) from v0.7.1 into main
Reviewed-on: #27
2026-07-17 03:28:17 +00:00
ton 39cf9a72a1 Merge pull request 'v0.7.1-fix_single_items_info_frontend' (#26) from v0.7.1-fix_single_items_info_frontend into v0.7.1
Reviewed-on: #26
2026-07-17 03:28:06 +00:00
ton 1c829ad854 update 2026-07-17 10:14:13 +07:00
ton da98baddb6 update 2026-07-17 10:06:32 +07:00
ton c5fbaabf42 Merge pull request 'v0.7.0' (#25) from v0.7.0 into main
Reviewed-on: #25
2026-07-17 00:03:01 +00:00
6 changed files with 753 additions and 350 deletions
+39 -3
View File
@@ -2,7 +2,7 @@
julia_version = "1.12.6"
manifest_format = "2.0"
project_hash = "1e317787f914f6d857feb7c23bb910d1185caed9"
project_hash = "dc7878808bbc4637a12e709dd495979a784824a5"
[[deps.Accessors]]
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
@@ -244,6 +244,12 @@ git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
version = "0.1.10"
[[deps.EzXML]]
deps = ["Printf", "XML2_jll"]
git-tree-sha1 = "7ea1aa5869e2626ccae84480e4f37185bc6f41d3"
uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615"
version = "1.2.3"
[[deps.FileIO]]
deps = ["Pkg", "Requires", "UUIDs"]
git-tree-sha1 = "6621fef488e496356c9c9625d0562c12a6070819"
@@ -500,6 +506,12 @@ version = "1.11.3+1"
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
version = "1.11.0"
[[deps.Libiconv_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809"
uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
version = "1.18.0+0"
[[deps.LinearAlgebra]]
deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -828,6 +840,12 @@ git-tree-sha1 = "084c47c7c5ce5cfecefa0a98dff69eb3646b5a80"
uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
version = "1.4.10"
[[deps.Serde]]
deps = ["CSV", "Dates", "EzXML", "JSON", "TOML", "UUIDs", "YAML"]
git-tree-sha1 = "f397fc8779cc53e4677c2708f3802c6996f28d00"
uuid = "db9b398d-9517-45f8-9a95-92af99003e0e"
version = "3.7.2"
[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
version = "1.11.0"
@@ -932,6 +950,12 @@ git-tree-sha1 = "cd83a04baf746e3b43b83c61b7de77ab0409b80a"
uuid = "88034a9c-02f8-509d-84a9-84ec65e18404"
version = "1.0.0"
[[deps.StringEncodings]]
deps = ["Libiconv_jll"]
git-tree-sha1 = "b765e46ba27ecf6b44faf70df40c57aa3a547dcb"
uuid = "69024149-9ee7-55f6-a4c4-859efe599b68"
version = "0.3.7"
[[deps.StringManipulation]]
deps = ["PrecompileTools"]
git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
@@ -1052,11 +1076,23 @@ git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7"
uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60"
version = "1.6.1"
[[deps.XML2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"]
git-tree-sha1 = "3f3315d89fc954a28f5b471bce698ed6e27481be"
uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
version = "2.15.3+0"
[[deps.YAML]]
deps = ["Base64", "Dates", "Printf", "StringEncodings"]
git-tree-sha1 = "a1c0c7585346251353cddede21f180b96388c403"
uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
version = "0.4.16"
[[deps.YiemAgent]]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SQLLLM", "Serialization", "URIs", "UUIDs"]
deps = ["Base64", "CSV", "DataFrames", "DataStructures", "Dates", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SQLLLM", "Serialization", "URIs", "UUIDs"]
path = "."
uuid = "e012c34b-7f78-48e0-971c-7abb83b6f0a2"
version = "0.6.5"
version = "0.7.2"
[[deps.Zlib_jll]]
deps = ["Libdl"]
+3 -1
View File
@@ -1,6 +1,6 @@
name = "YiemAgent"
uuid = "e012c34b-7f78-48e0-971c-7abb83b6f0a2"
version = "0.7.0"
version = "0.7.4"
authors = ["narawat lamaiin <narawat@outlook.com>"]
[deps]
@@ -19,6 +19,7 @@ PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
SQLLLM = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
Serde = "db9b398d-9517-45f8-9a95-92af99003e0e"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -33,3 +34,4 @@ JSON = "1.6.1"
LLMMCTS = "0.1.5"
NATS = "0.1.0"
SQLLLM = "0.2.8"
Serde = "3.7.2"
+89
View File
@@ -109,3 +109,92 @@ end
function generate_wine_retail_sql(conditions::Dict{String, Any})::String
# 1. Base SQL structure
base_query = """
SELECT
w.winery,
w.wine_name,
w.wine_id,
w.vintage,
w.region,
w.country,
w.wine_type,
w.grape,
w.serving_temperature,
w.sweetness,
w.intensity,
w.tannin,
w.acidity,
w.tasting_notes,
rw.price,
rw.currency,
w.image_url,
NULL AS retailer_name,
rw.retailer_id
FROM wine AS w
JOIN retailer_wine AS rw
ON w.wine_id = rw.wine_id
"""
# 2. Dynamic WHERE Clause Builder
where_clauses = String[]
# Iterate over each table condition provided
for (table_name, table_conditions) in conditions
# Determine table alias
alias = if table_name == "wine"
"w"
elseif table_name == "retailer_wine"
"rw"
else
continue # Skip unsupported tables
end
# Process condition dictionaries
if isa(table_conditions, Dict) && !isempty(table_conditions)
for (column_name, filter_details) in table_conditions
if isa(filter_details, Dict) && haskey(filter_details, "operator")
op = filter_details["operator"]
raw_val = filter_details["value"]
# --- Value Type Handling ---
# Use tryparse instead of try/catch for cleaner, faster parsing
final_val = raw_val
if op in ("=", "<", ">", "<=", ">=")
str_val = string(raw_val)
num_val = tryparse(Float64, str_val)
if !isnothing(num_val)
final_val = isinteger(num_val) ? round(Int, num_val) : num_val
end
end
# --- SQL Formatting ---
if isa(final_val, Number)
clause = "$(alias).$(column_name) $(op) $(final_val)"
else
# Escape single quotes within string values
escaped_val = replace(string(final_val), "'" => "''")
clause = "$(alias).$(column_name) $(op) '$(escaped_val)'"
end
push!(where_clauses, clause)
end
end
end
end
# 3. Assemble Final Query
where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ")
return string(base_query, where_sql, ";")
end
+39 -28
View File
@@ -4,7 +4,7 @@ export addNewMessage, conversation, decisionMaker, reflector, generatechat,
generalconversation, detectWineryName, generateSituationReport
using JSON, DataStructures, Dates, UUIDs, HTTP, Random, PrettyPrinting, Serialization,
DataFrames, CSV
DataFrames, Serde
using GeneralUtils
using ..type, ..util, ..llmfunction
@@ -73,7 +73,7 @@ OrderedDict{String, Any} with 4 entries:
"action_result" => "1) winery: Terrazze dell Etna, wine_name: Rose Brut.
```
"""
function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
function decisionMaker(a::T; recentevents::Integer=20, maxattempt=3
) where {T<:agent}
@info "YiemAgent decisionMaker() start " @__LINE__
# lessonDict = copy(JSON.parsefile("lesson.json"))
@@ -122,22 +122,16 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
errornote = "N/A"
response = nothing # placeholder for show when error msg show up
for attempt in 1:maxattempt
msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => a.chathistory,
"temperature" => 0.7
)
for attempt in 1:maxattempt
response = a.context.text2textInstructLLM(a.id, msg)
response = GeneralUtils.clean_json_response(response)
response = GeneralUtils.remove_french_accents(response)
think, response = GeneralUtils.extractthink(response)
response = String(split(response, ", observation")[1]) # in case LLM generate observation key which it isn't supposed to
response = strip(response)
# think, response = GeneralUtils.extractthink(response)
# dollar sign in Julia means string interpolation
while occursin('$', response)
@@ -145,16 +139,10 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
end
responsedict = nothing
if occursin(requiredKeys[2], response)
try
_responsedict = JSON.parse(response)
responsedict = GeneralUtils.dictify(_responsedict; keytype=String, sort_order=requiredKeys)
catch
println("\nERROR YiemAgent decisionMaker() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
end
else
println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)-> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
responsedict = Serde.parse_yaml(response)
catch e
println("\nERROR YiemAgent decisionMaker() Error: $e --(not qualify response)-> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
continue
end
@@ -187,7 +175,14 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=10
@info "YiemAgent decisionMaker() end " @__LINE__
return responsedict
end
error("DecisionMaker failed to generate a thought ", response)
# in case decisionMaker failed, force to use generatechat!()
responsedict = OrderedDict(
"plan"=> "N/A",
"action_name"=> "CHAT_BOX",
"action_input"=> "N/A"
)
return responsedict
end
@@ -386,8 +381,6 @@ function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Obj
if loopcount > max_think_loop
thoughtdict, result_raw = generatechat!(a)
assistant_response = Dict{String, Any}(
"role" => "assistant",
"content" => [Dict("type" => "text", "text" => thoughtdict["action_input"]),]
@@ -395,13 +388,19 @@ function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Obj
addNewMessage(a, "assistant", assistant_response; maximumMsg=maximumMsg)
items_info = []
send_item_ind = [] # index of the item being send to frontend
if haskey(a.memory["shortmem"], "items_info")
for (i, item) in enumerate(a.memory["shortmem"]["items_info"])
@info "YiemAgent conversation() shortmem: $(length(a.memory["shortmem"]["items_info"])), items_info: $(length(items_info)), item name: $(item["wine_name"]) " @__LINE__
if haskey(item, "wine_name") && occursin(item["wine_name"], thoughtdict["action_input"])
push!(items_info, item)
deleteat!(a.memory["shortmem"]["items_info"], i)
push!(items_info, deepcopy(item))
push!(send_item_ind, i)
@info "YiemAgent conversation() shortmem: $(length(a.memory["shortmem"]["items_info"])), items_info: $(length(items_info)) " @__LINE__
end
end
# remove sent items
deleteat!(a.memory["shortmem"]["items_info"], send_item_ind)
@info "YiemAgent conversation() shortmem: $(length(a.memory["shortmem"]["items_info"])), items_info: $(length(items_info)) " @__LINE__
end
response_to_frontend = Dict{String, Any}(
@@ -422,8 +421,6 @@ function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Obj
thoughtdict, result_raw = think(a)
if thoughtdict["action_name"] ["CHAT_BOX"]
assistant_response = Dict{String, Any}(
"role" => "assistant",
"content" => [Dict("type" => "text", "text" => thoughtdict["action_input"]),]
@@ -431,13 +428,19 @@ function conversation(a::sommelier; userinput::Union{Dict{String, Any}, JSON.Obj
addNewMessage(a, "assistant", assistant_response; maximumMsg=maximumMsg)
items_info = []
send_item_ind = [] # index of the item being send to frontend
if haskey(a.memory["shortmem"], "items_info")
for (i, item) in enumerate(a.memory["shortmem"]["items_info"])
@info "YiemAgent conversation() shortmem: $(length(a.memory["shortmem"]["items_info"])), items_info: $(length(items_info)), item name: $(item["wine_name"]) " @__LINE__
if haskey(item, "wine_name") && occursin(item["wine_name"], thoughtdict["action_input"])
push!(items_info, item)
deleteat!(a.memory["shortmem"]["items_info"], i)
push!(items_info, deepcopy(item))
push!(send_item_ind, i)
@info "YiemAgent conversation() shortmem: $(length(a.memory["shortmem"]["items_info"])), items_info: $(length(items_info)) " @__LINE__
end
end
# remove sent items
deleteat!(a.memory["shortmem"]["items_info"], send_item_ind)
@info "YiemAgent conversation() shortmem: $(length(a.memory["shortmem"]["items_info"])), items_info: $(length(items_info)) " @__LINE__
end
response_to_frontend = Dict{String, Any}(
@@ -527,7 +530,14 @@ function think(a::T)::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict,
result_raw = nothing
if thoughtdict["action_name"] ["CHAT_BOX"]
# sometime CHAT_BOX input is too short.
# if thoughtdict["action_input] < 20 character, use generatechat!()
if length(thoughtdict["action_input"]) < 20
thoughtdict, result_raw = generatechat!(a)
else
thoughtdict["action_result"] = "Action result is the next user dialogue."
result_raw = thoughtdict["action_input"]
end
elseif thoughtdict["action_name"] == "END_CONVER_GUIDELINE"
@@ -784,6 +794,7 @@ function generatechat!(a::T; maxattempt::Integer=10
@info "YiemAgent generatechat!() end " @__LINE__
return (thoughtdict=responsedict, result_raw=responsedict["action_input"])
end
@info "YiemAgent generatechat() failed to generate a thought " @__LINE__
error("YiemAgent generatechat() failed to generate a thought ", response)
end
+563 -225
View File
@@ -5,7 +5,7 @@ export virtualWineUserChatbox, jsoncorrection, search_wine_database!, # recomme
extractWineAttributes_2, paraphrase, SQLexecution
using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, Dates, DataFrames, DataStructures,
Base64
Base64, Serde, LibPQ
using GeneralUtils, SQLLLM
using ..type, ..util
@@ -277,24 +277,18 @@ end
# Example
```jldoctest
julia> using ChatAgent
julia> agent = ChatAgent.agentReflex("Jene")
julia> input = "{\"food\": \"pizza\", \"occasion\": \"anniversary\"}"
julia> result = checkinventory(agent, input)
"{"wine 1": {\"Winery\": \"Pichon Baron\", \"wine name\": \"Pauillac (Grand Cru Classé)\", \"grape variety\": \"Cabernet Sauvignon\", \"year\": 2010, \"price\": \"125 USD\", \"stock ID\": \"ar-17\"}, }"
julia> agent = YiemAgent.sommelier(...)
julia> thoughtdict =
OrderedDict{String, Any}(
"plan" => "The user is asking a very specific question about a wine (Brunello di Montalcino from Tenuta CastelGiocondo). Although the policy suggests gathering budget, wine type, and occasion, the user has provided enough specific information (name, region, producer) to attempt a direct search in the database. I will use the SEARCH_WINE_DATABASE action to check if this specific wine is in our inventory.",
"action_name" => "SEARCH_WINE_DATABASE",
"action_input" => "Brunello di Montalcino from Tenuta CastelGiocondo")
```
"""
function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false
)::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent}
println("\ncheckinventory order: $(thoughtdict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
wineattributes_1 = extractWineAttributes_1(a, thoughtdict["action_input"])
wineattributes_2 = extractWineAttributes_2(a, thoughtdict["action_input"])
retrieve_attributes = ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"]
_inventoryquery = "$(thoughtdict["action_input"]), $wineattributes_1, $wineattributes_2, retailer_name: $(a.retailername), retailerid: $(a.retailerid)"
inventoryquery = "Retrieves $retrieve_attributes of wines that match the following criteria - {$_inventoryquery}"
println("\ncheckinventory input: $inventoryquery ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
if useSQLLLM
# add suppport for similarSQLVectorDB
textresult, result_raw = SQLLLM.query(
@@ -308,7 +302,8 @@ function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=
else
# direct query with possible sql instead of SQLLLM.
sql = generatesql(a, inventoryquery)
sql = predefined_wine_search_sql(a, thoughtdict["action_input"])
# sql = generatesql(a, inventoryquery)
println("\nSQL: $sql ", @__FILE__, ":", @__LINE__, " $(Dates.now()) \n")
textresult, sql_result_df, success, _ = SQLexecution(a.context.executeSQL, sql)
@@ -366,9 +361,9 @@ function generatesql(a::T, searchterm::String,
Fulfill the objective.
# you should then respond to the user with interleaving plan, action_name, action_input
1) "plan, Based on the current situation, state a complete action plan to complete the task and rationale. Be specific.
2) "action_name, Must be "RUNSQL"
3) "action_input, The input to the action you are about to perform according to your plan.
1) "plan", Based on the current situation, state a complete action plan to complete the task and rationale. Be specific.
2) "action_name", Must be "RUNSQL"
3) "action_input", The input to the action you are about to perform according to your plan.
After the action is executed you gets "action_result". It is the output from the action you selected.
# you should only respond in JSON format as described below
@@ -610,6 +605,203 @@ function generatesql(a::T, searchterm::String,
error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
end
"""
# Example
```jldoctest
julia> using ChatAgent
julia> agent = YiemAgent.sommelier(...)
julia> thoughtdict =
OrderedDict{String, Any}(
"plan" => "The user is asking a very specific question about a wine (Brunello di Montalcino from Tenuta CastelGiocondo). Although the policy suggests gathering budget, wine type, and occasion, the user has provided enough specific information (name, region, producer) to attempt a direct search in the database. I will use the SEARCH_WINE_DATABASE action to check if this specific wine is in our inventory.",
"action_name" => "SEARCH_WINE_DATABASE",
"action_input" => "Brunello di Montalcino from Tenuta CastelGiocondo")
```
julia> predefined_wine_search_sql(agent, thoughtdict["action_input"])
"""
function predefined_wine_search_sql(a::T, searchterm::String,
; maxattempt=10
)::String where {T<:agent}
systemmsg =
"""
# situation
At each round of conversation, you will be given the following:
- user search term
- database tables schema
# objective
Consult the provided database schema (tables and columns), please map a user's natural-language search term to the appropriate database columns and tables—identify the relevant fields, operators, and values (e.g., for SQL filtering).
# your responsibility includes
Fulfill the objective.
# you should only respond in YAML format as described below
table_name_1:
column_name_1:
operator: "="
value: "..."
column_name_2:
operator: "="
value: "..."
...
table_name_2:
column_name_1:
operator: "="
value: "..."
column_name_2:
operator: "="
value: "..."
...
# here are some example
<user>
4-wheel drive car with red color that will give me fast and furious emotion. No more than 7000 USD
</user>
<assistant>
car_info: # table_name
drive_type: # column_name
operator: "=" # operator is not "N/A" because drive_type column store quantitative value
value: "4-wheel" # column_value
color:
operator: "=" # operator is not "N/A" because color column store quantitative value
value: "red"
drive_feeling:
operator: "N/A" # operator is "N/A" because drive_feeling column store qualitative value
value: "fast and furious"
price_list:
price:
operator: "<" # operator is not "N/A" because drive_type column store quantitative value
value: "7000"
</assistant>
"""
# use find_related_tables_for_user_question and inject only related table schema for a given search term
# to LLM instead of giving LLM all tables schema.
related_tables = a.context.find_related_tables_for_user_question(searchterm)
table_schema = ""
for table in related_tables
_table_schema_str = get_db_table_schema_simple_with_samples(a.context.pg_conn_str, table)
# _table_schema_str = GeneralUtils.get_db_table_schema_simple(a.context.pg_conn_str, table)
table_schema_str = sprint(show, _table_schema_str) * "\n"
table_schema = table_schema * table_schema_str
end
context =
"""
<internal_context_for_assistant>
<database_table_schema>
$table_schema
</database_table_schema>
</internal_context_for_assistant>
"""
input = context * searchterm
msg = Dict(
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
"messages" => [
Dict(
"role" => "system",
"content" => [
Dict("type" => "text", "text" => systemmsg),
]
),
Dict(
"role" => "user",
"content" => [
Dict("type" => "text", "text" => input),
]
),
],
"temperature" => 0.7
)
for attempt in 1:maxattempt
response = a.context.text2textInstructLLM("random_id", msg)
responsedict = nothing
try
responsedict = Serde.parse_yaml(response)
catch e
println("\nERROR YiemAgent predefined_wine_search_sql() Error: $e --(not qualify response)-> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
continue
end
# println("\n ", table_schema)
println("\n ", responsedict)
@info "before BM25 " @__LINE__
"""
responsedict = Dict(
"wine" => Dict(
"tasting_notes" => Dict(
"operator" => "N/A",
"value" => "casual dinner"
),
"wine_type" => Dict(
"operator" => "=",
"value" => "red"
)
),
"retailer_wine" => Dict(
"currency" => Dict(
"operator" => "=", "value" => "USD"
),
"price" => Dict(
"operator" => "<", "value" => "1000"
)
)
)
"""
for (table_name, table_info_dict) in responsedict
for (column_name, v) in table_info_dict
bucket = classify_column(a.context.pg_conn_str, table_name, column_name)
if bucket == "fuzzy_correction"
words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, table_name, column_name)
resolved_word = GeneralUtils.resolve_entity(v["value"], words_catalog; threshold=0.9)
table_info_dict[column_name]["value"] = resolved_word
end
end
end
# filter for column that will be used for hard condition (SQL where clause)
# column with "N/A" operator will be used in vector search
vector_search_words = ""
for (table_name, table_dict) in responsedict
for (column_name, column_dict) in table_dict
if column_dict["operator"] ["=","<>","!=",">","<",">=","<=","!<","!>","<=>"]
vector_search_words = vector_search_words * column_dict["value"] * ", "
delete!(table_dict, column_name)
# remove table from responsedict if there is no column to used
if length(responsedict[table_name]) == 0
delete!(responsedict, table_name)
end
end
end
end
println("")
pprintln(responsedict)
@info "predefined_wine_search_sql() " @__LINE__
#WORKING do vector searched
println("")
@show vector_search_words
sql = predefined_wine_search_sql(responsedict)
return sql
end
error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
end
function SQLexecution(executeSQL::Function, sql::T
)::NamedTuple where {T<:AbstractString}
@@ -660,6 +852,64 @@ function SQLexecution(executeSQL::Function, sql::T
end
end
function DEPRECIATED_search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false
)::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent}
# XXX
predefined_wine_search_sql(a, thoughtdict["action_input"])
println("\ncheckinventory order: $(thoughtdict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
wineattributes_1 = extractWineAttributes_1(a, thoughtdict["action_input"])
wineattributes_2 = extractWineAttributes_2(a, thoughtdict["action_input"])
retrieve_attributes = ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"]
_inventoryquery = "$(thoughtdict["action_input"]), $wineattributes_1, $wineattributes_2, retailer_name: $(a.retailername), retailerid: $(a.retailerid)"
inventoryquery = "Retrieves $retrieve_attributes of wines that match the following criteria - {$_inventoryquery}"
println("\ncheckinventory input: $inventoryquery ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
if useSQLLLM
# add suppport for similarSQLVectorDB
textresult, result_raw = SQLLLM.query(
inventoryquery,
a.context.executeSQL,
a.context.text2textInstructLLM;
insertSQLVectorDB=a.context.insertSQLVectorDB,
similarSQLVectorDB=a.context.similarSQLVectorDB,
llmFormatName="qwen3")
thoughtdict["action_result"] = textresult
else
# direct query with possible sql instead of SQLLLM.
sql = generatesql(a, inventoryquery)
println("\nSQL: $sql ", @__FILE__, ":", @__LINE__, " $(Dates.now()) \n")
textresult, sql_result_df, success, _ = SQLexecution(a.context.executeSQL, sql)
items = nothing
if sql_result_df !== nothing
result_vec = GeneralUtils.dfToVectorDict(sql_result_df)
# get image
for d in result_vec
image_url_json_str = d["image_url"]
image_url_json_obj = JSON.parse(image_url_json_str)
base_url = "http://192.168.88.106:8080/"
if haskey(image_url_json_obj, "bottle")
url = base_url * image_url_json_obj["bottle"]
image_data = HTTP.get(url) # vector{int} data
image_base64_string = base64encode(image_data.body)
d["image"] = image_base64_string
else
d["image"] = nothing
end
end
items = result_vec # image is added to each item
end
thoughtdict["action_result"] = textresult
end
return (thoughtdict=thoughtdict, result_raw=items)
end
"""
@@ -1036,213 +1286,183 @@ function extractWineAttributes_2(a::T1, input::T2)::String where {T1<:agent, T2<
error("extractWineAttributes_2() failed to get a response")
end
function paraphrase(text2textInstructLLM::Function, text::String)
systemmsg =
function predefined_wine_search_sql(conditions::Dict{String, Any})::String
# 1. Base SQL structure
base_query =
"""
Your name: N/A
Your vision:
- You are a helpful assistant who help the user to paraphrase their text.
Your mission:
- To help paraphrase the user's text
Mission's objective includes:
- To help paraphrase the user's text
Your responsibility includes:
1) To help paraphrase the user's text
Your responsibility does NOT includes:
1) N/A
Your profile:
- N/A
Additional information:
- N/A
At each round of conversation, you will be given the following information:
Text: The user's given text
You MUST follow the following guidelines:
- N/A
You should follow the following guidelines:
- N/A
You should then respond to the user with:
Paraphrase: Paraphrased text
You should only respond in format as described below:
Paraphrase: ...
Let's begin!
"""
#[PENDING] use JSON the same as extractWineAttributes_1 is better. change this function to use the same format use decisionMaker
header = ["Paraphrase:"]
dictkey = ["paraphrase"]
errornote = "N/A"
response = nothing # placeholder for show when error msg show up
for attempt in 1:10
usermsg = """
Text: $text
P.S. $errornote
SELECT
w.winery,
w.wine_name,
w.wine_id,
w.vintage,
w.region,
w.country,
w.wine_type,
w.grape,
w.serving_temperature,
w.sweetness,
w.intensity,
w.tannin,
w.acidity,
w.tasting_notes,
rw.price,
rw.currency,
w.image_url,
r.retailer_name,
rw.retailer_id
FROM wine AS w
JOIN retailer_wine AS rw ON w.wine_id = rw.wine_id
JOIN retailer AS r ON rw.retailer_id = r.retailer_id
"""
_prompt =
[
Dict("name" => "system", "text" => systemmsg),
Dict("name" => "user", "text" => usermsg)
]
# 2. Dynamic WHERE Clause Builder
where_clauses = String[]
# put in model format
prompt = GeneralUtils.formatLLMtext(_prompt, a.llmFormatName)
# Iterate over each table condition provided
for (table_name, table_conditions) in conditions
try
response = text2textInstructLLM(prompt)
response = GeneralUtils.deFormatLLMtext(response, a.llmFormatName)
think, response = GeneralUtils.extractthink(response)
# sometime the model response like this "here's how I would respond: ..."
if occursin("respond:", response)
errornote = "You don't need to intro your response"
error("\nparaphrase() response contain : ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
response = GeneralUtils.remove_french_accents(response)
response = replace(response, '*'=>"")
response = replace(response, '$' => "USD")
response = replace(response, '`' => "")
response = GeneralUtils.remove_french_accents(response)
# check whether response has all answer's key points
detected_kw = GeneralUtils.detect_keyword(header, response)
if 0 values(detected_kw)
errornote = "\nYiemAgent paraphrase() response does not have all answer's key points"
continue
elseif sum(values(detected_kw)) > length(header)
errornote = "\nnYiemAgent paraphrase() response has duplicated answer's key points"
continue
end
responsedict = GeneralUtils.textToDict(response, header;
dictKey=dictkey, symbolkey=true)
for i [:paraphrase]
if length(JSON.json(responsedict[i])) == 0
error("$i is empty ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
end
# check if there are more than 1 key per categories
for i [:paraphrase]
matchkeys = GeneralUtils.findMatchingDictKey(responsedict, i)
if length(matchkeys) > 1
error("paraphrase() has more than one key per categories")
end
end
println("\nparaphrase() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
pprintln(Dict(responsedict))
result = responsedict["paraphrase"]
return result
catch e
io = IOBuffer()
showerror(io, e)
errorMsg = String(take!(io))
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
println("\nAttempt $attempt. Error occurred: $errorMsg\n$st ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
end
end
error("paraphrase() failed to generate a response")
end
""" Attemp to correct LLM response's incorrect JSON response.
# Arguments
- `a::T1`
one of Yiem's agent
- `input::T2`
text to be send to virtual wine customer
# Return
- `correctjson::String`
corrected json string
# Example
```jldoctest
julia>
```
# Signature
"""
function jsoncorrection(config::T1, input::T2, correctJsonExample::T3;
maxattempt::Integer=3
) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
incorrectjson = deepcopy(input)
correctjson = nothing
for attempt in 1:maxattempt
try
d = copy(JSON.parsefile(incorrectjson))
correctjson = incorrectjson
return correctjson
catch e
@warn "Attempting to correct JSON string. Attempt $attempt"
e = """$e"""
if occursin("EOF", e)
e = split(e, "EOF")[1] * "EOF"
end
incorrectjson = deepcopy(input)
_prompt =
"""
Your goal are:
1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function.
2) Provide Corrected JSON string only. Do not provide any other info.
$correctJsonExample
Let's begin!
Given JSON string: $incorrectjson
The given JSON string failed to load previously because: $e
Corrected JSON string:
"""
# apply LLM specific instruct format
externalService = config["externalservice"]["text2textinstruct"]
llminfo = externalService["llminfo"]
prompt =
if llminfo["name"] == "llama3instruct"
formatLLMtext_llama3instruct("system", _prompt)
# Determine table alias
alias = if table_name == "wine"
"w"
elseif table_name == "retailer_wine"
"rw"
else
error("llm model name is not defied yet $(@__LINE__)")
continue # Skip unsupported tables
end
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
msgMeta = GeneralUtils.generate_msgMeta(
externalService["mqtttopic"],
senderName= "jsoncorrection",
senderId= string(uuid4()),
receiverName= "text2textinstruct",
mqttBroker= config["mqttServerInfo"]["broker"],
mqttBrokerPort= config["mqttServerInfo"]["port"],
)
# Process condition dictionaries
if isa(table_conditions, Dict) && !isempty(table_conditions)
outgoingMsg = Dict(
"msgMeta"=> msgMeta,
"payload"=> Dict(
"text"=> prompt,
"kwargs"=> Dict(
"max_tokens"=> 512,
"stop"=> ["<|eot_id|>"],
)
)
)
result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
incorrectjson = result[:response][:text]
for (column_name, filter_details) in table_conditions
if isa(filter_details, Dict) && haskey(filter_details, "operator")
op = filter_details["operator"]
raw_val = filter_details["value"]
# --- Value Type Handling ---
# Use tryparse instead of try/catch for cleaner, faster parsing
final_val = raw_val
if op in ("=", "<", ">", "<=", ">=")
str_val = string(raw_val)
num_val = tryparse(Float64, str_val)
if !isnothing(num_val)
final_val = isinteger(num_val) ? round(Int, num_val) : num_val
end
end
# --- SQL Formatting ---
if isa(final_val, Number)
clause = "$(alias).$(column_name) $(op) $(final_val)"
else
# Escape single quotes within string values
escaped_val = replace(string(final_val), "'" => "''")
clause = "$(alias).$(column_name) $(op) '$(escaped_val)'"
end
push!(where_clauses, clause)
end
end
end
end
# 3. Assemble Final Query
where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ")
return string(base_query, where_sql, ";")
end
function get_db_table_schema_simple_with_samples(pg_conn_str::String, table_name::String;
schema_name::String="public")::String
conn = LibPQ.Connection(pg_conn_str)
return get_db_table_schema_simple_with_samples(conn, table_name; schema_name=schema_name)
end
function get_db_table_schema_simple_with_samples(conn, table_name::String; schema_name::String="public", sample_count::Int=3)::String
# 1. SQL query for catalog metadata
meta_sql = """
SELECT
a.attname AS column_name,
format_type(a.atttypid, a.atttypmod) AS data_type,
pg_get_expr(def.adbin, def.adrelid) AS default_value,
COALESCE(
(SELECT pg_get_constraintdef(p.oid)
FROM pg_catalog.pg_constraint p
WHERE p.conrelid = c.oid AND a.attnum = ANY(p.conkey)
LIMIT 1), ''
) AS constraint_definition
FROM pg_catalog.pg_attribute a
JOIN pg_catalog.pg_class c ON a.attrelid = c.oid
JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
LEFT JOIN pg_catalog.pg_attrdef def ON def.adrelid = c.oid AND def.adnum = a.attnum
WHERE c.relname = \$1
AND n.nspname = \$2
AND a.attnum > 0
AND NOT a.attisdropped
ORDER BY a.attnum;
"""
meta_res = DataFrame(execute(conn, meta_sql, [table_name, schema_name]))
if nrow(meta_res) == 0
error("Table '$schema_name.$table_name' not found.")
end
# 2. Build single dynamic query to fetch non-null samples for all columns
sample_selects = String[]
for row in eachrow(meta_res)
c_name = row.column_name
push!(sample_selects, """
(SELECT json_agg(s."$c_name")
FROM (
SELECT "$c_name"
FROM "$schema_name"."$table_name"
WHERE "$c_name" IS NOT NULL
LIMIT $sample_count
) s
) AS "$c_name"
""")
end
sample_sql = "SELECT " * join(sample_selects, ",\n ") * ";"
sample_df = DataFrame(execute(conn, sample_sql))
# 3. Build DDL definitions with inline sample comments
ddl_lines = String[]
constraints = String[]
for row in eachrow(meta_res)
col_name = row.column_name
data_type = row.data_type
default_val = ismissing(row.default_value) ? "" : " DEFAULT " * row.default_value
col_def = " \"$col_name\" $data_type$default_val"
# Fetch sample data for this column from the single-row sample DataFrame
samples_comment = ""
if nrow(sample_df) > 0
raw_samples = sample_df[1, Symbol(col_name)]
samples_str = ismissing(raw_samples) || isnothing(raw_samples) ? "[]" : string(raw_samples)
samples_comment = " -- Samples: $samples_str"
end
push!(ddl_lines, col_def * samples_comment)
# Handle table-level constraints
con_def = ismissing(row.constraint_definition) ? "" : row.constraint_definition
if !isempty(con_def) && !(con_def in constraints)
push!(constraints, " " * con_def)
end
end
all_definitions = vcat(ddl_lines, constraints)
body = join(all_definitions, ",\n")
return "CREATE TABLE \"$schema_name\".\"$table_name\" (\n$body\n);"
end
@@ -1254,27 +1474,145 @@ end
function classify_column(pg_conn_str::String, table_name::String, column_name::String;
sample_size::Integer=1000)
conn = LibPQ.Connection(pg_conn_str)
return classify_column(conn, table_name, column_name; sample_size=sample_size)
end
function classify_column(conn::LibPQ.Connection, table_name::String, column_name::String; sample_size::Int=1000)
# 1. Fetch BOTH data_type and udt_name (User Defined Type name)
meta_query = """
SELECT data_type, udt_name
FROM information_schema.columns
WHERE table_name = lower('$(table_name)')
AND column_name = lower('$(column_name)');
"""
pg_type = "unknown"
udt_name = "unknown"
try
df = DataFrame(LibPQ.execute(conn, meta_query))
if !isempty(df)
pg_type = df[1, :data_type]
udt_name = df[1, :udt_name]
end
catch e
@error "Failed to fetch metadata for $table_name.$column_name" exception=e
return "error"
end
# 2. FAST-TRACK: Check for pgvector FIRST
# pgvector registers as "USER-DEFINED" in data_type, but "vector" in udt_name
if udt_name == "vector"
return "semantic_search"
end
# 3. FAST-TRACK: Hard rules for standard non-text Postgres types
if pg_type in ["integer", "bigint", "smallint", "numeric", "real",
"double precision", "boolean", "date",
"timestamp without time zone", "timestamp with time zone", "uuid"]
return "exact_or_range"
end
# 4. SAMPLE: Get text statistics for remaining text columns
stats_query = """
SELECT
COUNT(*)::int AS total_count,
COUNT(DISTINCT $(column_name)::text)::int AS unique_count,
COALESCE(AVG(LENGTH($(column_name)::text)), 0)::float AS avg_len,
COALESCE(STDDEV(LENGTH($(column_name)::text)), 0)::float AS std_len
FROM (
SELECT $(column_name)
FROM $(table_name)
WHERE $(column_name) IS NOT NULL
LIMIT $sample_size
) AS sampled_data;
"""
try
df = DataFrame(LibPQ.execute(conn, stats_query))
if isempty(df) || df[1, :total_count] == 0
return "unknown"
end
total = df[1, :total_count]
unique = df[1, :unique_count]
avg_len = df[1, :avg_len]
std_len = df[1, :std_len]
ratio = unique / total
# 5. HEURISTICS: Route the column_name to the correct text bucket
return classify_text_column(unique, ratio, avg_len, std_len)
catch e
@warn "Failed to sample column_name $table_name.$column_name" exception=e
return "unknown"
end
end
# The Decision Tree for Text Columns (Unchanged, but kept for completeness)
function classify_text_column(unique_count::Integer, ratio::Float64, avg_len::Float64, std_len::Float64)
if avg_len > 60 && std_len > 25
return "full_text_search"
end
if ratio > 0.90 && avg_len < 40
return "exact_or_regex"
end
if unique_count <= 100
return "fuzzy_correction"
end
if ratio > 0.10 && avg_len < 35
return "fuzzy_correction"
end
if avg_len < 60
return "fuzzy_correction"
end
return "full_text_search"
end
function harvest_entity_catalog(pg_conn_str::String, table::String, column::String)
conn = LibPQ.Connection(pg_conn_str)
return harvest_entity_catalog(conn, table, column)
end
function harvest_entity_catalog_with_pg_type(conn::LibPQ.Connection, table::String, column::String)
try
# 1. Query the actual data
data_query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;"
df = DataFrame(LibPQ.execute(conn, data_query))
values = String.(strip.(string.(df[!, 1])))
# 2. Query the database schema for the column's data type
# Note: Postgres stores unquoted table/column names in lowercase
type_query = """
SELECT data_type
FROM information_schema.columns
WHERE table_name = lower('$(table)')
AND column_name = lower('$(column)');
"""
type_df = DataFrame(LibPQ.execute(conn, type_query))
pg_type = isempty(type_df) ? "unknown" : type_df[1, 1]
return (values = values, type = pg_type)
catch e
@error "Failed to harvest catalog" exception=e
return (values = String[], type = "unknown")
finally
close(conn)
end
end
# Usage:
# result = harvest_entity_catalog_with_pg_type(conn, "users", "created_at")
# println(result.values) # ["2023-01-01", "2023-02-15"]
# println(result.type) # "timestamp without time zone"
+4 -77
View File
@@ -23,75 +23,6 @@ end
abstract type agent end
mutable struct companion <: agent
name::String # agent name
id::String # agent id
systemmsg::String # system message
tools::Dict # tools
maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized
chathistory::Vector{Dict{String, Any}}
memory::Dict{String, Any}
context::NamedTuple # NamedTuple of functions
llmFormatName::String
end
function companion(
context::agentcontext # NamedTuple of functions
;
name::String= "Assistant",
id::String= GeneralUtils.uuid4snakecase(),
maxHistoryMsg::Integer= 20,
chathistory::Vector{Dict{String, String}} = Vector{Dict{String, String}}(),
llmFormatName::String= "granite3",
systemmsg::String=
"""
Your name: $name
Your sex: Female
Your role: You are a helpful assistant.
You should follow the following guidelines:
- Focus on the latest conversation.
- Your like to be short and concise.
Let's begin!
""",
)
tools = Dict( # update input format
"CHAT_BOX"=> Dict(
"description" => "- CHAT_BOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.",
),
)
""" Memory
Ref: Chat prompt format https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/discussions/3
NO "system" message in chathistory because I want to add it at the inference time
chathistory= [
Dict("name"=>"user", "text"=> "Wassup!", "timestamp"=> Dates.now()),
Dict("name"=>"assistant", "text"=> "Hi I'm your assistant.", "timestamp"=> Dates.now()),
]
"""
memory = Dict{String, Any}(
"events"=> Vector{Dict{String, Any}}(),
"state"=> Dict{String, Any}(), # state of the agent
"recap"=> OrderedDict{String, Any}(), # recap summary of the conversation
)
newAgent = companion(
name,
id,
systemmsg,
tools,
maxHistoryMsg,
chathistory,
memory,
context,
llmFormatName
)
return newAgent
end
mutable struct sommelier <: agent
name::String # agent name
id::String # agent id
@@ -210,11 +141,7 @@ function sommelier(
memory = Dict{String, Any}(
"shortmem"=> OrderedDict{String, Any}(),
"scratchpad"=> "",
"events"=> Vector{Dict{String, Any}}(),
"state"=> Dict{String, Any}(
),
"recap"=> OrderedDict{String, Any}(),
)
newAgent = sommelier(
@@ -279,10 +206,10 @@ function sommelier(
3) "action_input", The input to the action you are about to perform according to your plan.
After the action is executed you gets "action_result". It is the output from the action you selected.
# you should only respond in JSON format as described below (not Markdown format)
"plan": "...",
"action_name": "...",
"action_input": "..."
# you should only respond in YAML format as described below
plan: "..."
action_name: "..."
action_input: "..."
# available actions
"CHAT_BOX", which you can use to talk with the user. The input is dialogue you want to chat with the user according to your plan.