1925 lines
70 KiB
Julia
1925 lines
70 KiB
Julia
module llmfunction
|
||
|
||
export virtualWineUserChatbox, jsoncorrection, search_wine_database!, # recommendbox,
|
||
virtualWineUserRecommendbox, userChatbox, userRecommendbox, extractWineAttributes_1,
|
||
extractWineAttributes_2, paraphrase, SQLexecution
|
||
|
||
using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, Dates, DataFrames, DataStructures,
|
||
Base64, LibPQ, NATS
|
||
using GeneralUtils, SQLLLM
|
||
using ..type, ..util
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
|
||
"""
|
||
Chat with a virtual wine customer for recommendations.
|
||
|
||
Formats the input for the configured LLM model and communicates via MQTT
|
||
to receive a response tuple containing text, selection, reward, and terminal status.
|
||
|
||
# Arguments
|
||
- `a::T1`: An agent instance (subtype of `agent`) with `config` containing `externalservice`
|
||
and `mqttServerInfo` keys
|
||
- `input::T2`: Text to send to the virtual wine customer LLM
|
||
|
||
# Returns
|
||
- `Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}}`: A tuple of
|
||
`(response_text, select, reward, isterminal)` where `select` may be `Nothing`
|
||
|
||
# Notes
|
||
- Requires `a.config["externalservice"]["virtualWineCustomer_1"]` with `llminfo` and `mqtttopic`.
|
||
- Requires `a.config["mqttServerInfo"]` with `broker` and `port`.
|
||
- Only supports `llama3instruct` model name (other models throw an error).
|
||
- Uses `GeneralUtils.sendReceiveMqttMsg` with a 120-second timeout.
|
||
|
||
# TODO
|
||
- Add `recommend()` to compare wines
|
||
"""
|
||
function virtualWineUserRecommendbox(a::T1, input
|
||
)::Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}} where {T1<:agent}
|
||
|
||
# put in model format
|
||
virtualWineCustomer = a.config["externalservice"]["virtualWineCustomer_1"]
|
||
llminfo = virtualWineCustomer["llminfo"]
|
||
prompt =
|
||
if llminfo["name"] == "llama3instruct"
|
||
formatLLMtext_llama3instruct("assistant", input)
|
||
else
|
||
error("llm model name is not defied yet $(@__LINE__)")
|
||
end
|
||
|
||
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
|
||
msgMeta = GeneralUtils.generate_msgMeta(
|
||
virtualWineCustomer["mqtttopic"],
|
||
senderName= "virtualWineUserRecommendbox",
|
||
senderId= a.id,
|
||
receiverName= "virtualWineCustomer",
|
||
mqttBroker= a.config["mqttServerInfo"]["broker"],
|
||
mqttBrokerPort= a.config["mqttServerInfo"]["port"],
|
||
msgId = "dummyid" #CHANGE remove after testing finished
|
||
)
|
||
|
||
outgoingMsg = Dict(
|
||
"msgMeta"=> msgMeta,
|
||
"payload"=> Dict(
|
||
"text"=> prompt,
|
||
)
|
||
)
|
||
|
||
result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
|
||
response = result["response"]
|
||
|
||
return (response["text"], response["select"], response["reward"], response["isterminal"])
|
||
end
|
||
|
||
|
||
|
||
"""
|
||
Chatbox for conversing with a virtual wine customer AI.
|
||
|
||
Formats the chat history with a system prompt, sends it via MQTT to a text2text instruct
|
||
LLM service, and parses the JSON response. Retries up to 5 times on failure.
|
||
|
||
# Arguments
|
||
- `config::T1`: Configuration dictionary (subtype of `AbstractDict`) containing:
|
||
- `externalservice["text2text"]["mqtttopic"]`: MQTT topic for the LLM service
|
||
- `mqttServerInfo["broker"]`: MQTT broker address
|
||
- `mqttServerInfo["port"]`: MQTT broker port
|
||
- `input::T2`: Current sommelier message text (subtype of `AbstractString`)
|
||
- `virtualCustomerChatHistory`: Chat history vector of dictionaries with `"name"` and `"text"` keys
|
||
|
||
# Returns
|
||
- `Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}}`: A tuple of
|
||
`(text, select, reward, isterminal)` representing the virtual customer's response
|
||
|
||
# Notes
|
||
- The system prompt defines the virtual customer's persona and response format.
|
||
- Chat history role names are transformed: "user" → "you", "assistant" → "sommelier".
|
||
- Uses `jsoncorrection` to fix malformed LLM JSON responses.
|
||
- Retries up to 5 times on error before throwing.
|
||
- Uses `formatLLMtext` with `"llama3instruct"` format.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> result = YiemAgent.virtualWineUserChatbox(config, sommelier_msg, history)
|
||
("I'd like something under $50", nothing, 0, false)
|
||
```
|
||
"""
|
||
function virtualWineUserChatbox(config::T1, input::T2, virtualCustomerChatHistory
|
||
)::Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}} where {T1<:AbstractDict, T2<:AbstractString}
|
||
|
||
previouswines =
|
||
"""
|
||
You have the following wines previously:
|
||
|
||
"""
|
||
|
||
systemmsg =
|
||
"""
|
||
You find yourself in a well-stocked wine store, engaged in a conversation with the store's knowledgeable sommelier.
|
||
You're on a quest to find a bottle of wine that aligns with your specific preferences and requirements.
|
||
|
||
The ideal wine you're seeking should meet the following criteria:
|
||
1. It should fit within your budget.
|
||
2. It should be suitable for the occasion you're planning.
|
||
3. It should pair well with the food you intend to serve.
|
||
4. It should be of a particular type of wine you prefer.
|
||
5. It should possess certain characteristics, including:
|
||
- The level of sweetness.
|
||
- The intensity of its flavor.
|
||
- The amount of tannin it contains.
|
||
- Its acidity level.
|
||
|
||
Here's the criteria details:
|
||
{
|
||
"budget": 50,
|
||
"occasion": "graduation ceremony",
|
||
"food pairing": "Thai food",
|
||
"type of wine": "red",
|
||
"wine sweetness level": "dry",
|
||
"wine intensity level": "full-bodied",
|
||
"wine tannin level": "low",
|
||
"wine acidity level": "medium",
|
||
}
|
||
|
||
You should only respond with "text", "select", "reward", "isterminal" steps.
|
||
"text" is your conversation.
|
||
"select" is an integer. Choose an option when presented with choices, or leave it null if none of the options satisfy you or if no choices are available.
|
||
"reward" is an integer, it can be three number:
|
||
1) 1 if you find the right wine.
|
||
2) 0 if you don’t find the ideal wine.
|
||
3) -1 if you’re dissatisfied with the sommelier’s response.
|
||
"isterminal" can be false if you still want to talk with the sommelier, true otherwise.
|
||
|
||
You should only respond in JSON format as describe below:
|
||
{
|
||
"text": "your conversation",
|
||
"select": null,
|
||
"reward": 0,
|
||
"isterminal": false
|
||
}
|
||
|
||
Here are some examples:
|
||
|
||
sommelier: "What's your budget?
|
||
you:
|
||
{
|
||
"text": "My budget is 30 USD.",
|
||
"select": null,
|
||
"reward": 0,
|
||
"isterminal": false
|
||
}
|
||
|
||
sommelier: "The first option is Zena Crown and the second one is Buano Red."
|
||
you:
|
||
{
|
||
"text": "I like the 2nd option.",
|
||
"select": 2,
|
||
"reward": 1,
|
||
"isterminal": true
|
||
}
|
||
|
||
Let's begin!
|
||
"""
|
||
|
||
pushfirst!(virtualCustomerChatHistory, Dict("name"=> "system", "text"=> systemmsg))
|
||
|
||
# replace the :user key in chathistory to allow the virtual wine customer AI roleplay
|
||
chathistory::Vector{Dict{String, Any}} = Vector{Dict{String, Any}}()
|
||
for i in virtualCustomerChatHistory
|
||
newdict = Dict()
|
||
newdict["name"] =
|
||
if i["name"] == "user"
|
||
"you"
|
||
elseif i["name"] == "assistant"
|
||
"sommelier"
|
||
else
|
||
i["name"]
|
||
end
|
||
|
||
newdict["text"] = i["text"]
|
||
push!(chathistory, newdict)
|
||
end
|
||
|
||
push!(chathistory, Dict("name"=> "assistant", "text"=> input))
|
||
|
||
# put in model format
|
||
prompt = formatLLMtext(chathistory, "llama3instruct")
|
||
prompt *=
|
||
"""
|
||
<|start_header_id|>you<|end_header_id|>
|
||
{"text"
|
||
"""
|
||
|
||
pprint(prompt)
|
||
externalService = config["externalservice"]["text2textinstruct"]
|
||
|
||
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
|
||
msgMeta = GeneralUtils.generate_msgMeta(
|
||
externalService["mqtttopic"],
|
||
senderName= "virtualWineUserChatbox",
|
||
senderId= string(uuid4()),
|
||
receiverName= "text2textinstruct",
|
||
mqttBroker= config["mqttServerInfo"]["broker"],
|
||
mqttBrokerPort= config["mqttServerInfo"]["port"],
|
||
msgId = string(uuid4()) # remove after testing finished
|
||
)
|
||
|
||
outgoingMsg = Dict(
|
||
"msgMeta"=> msgMeta,
|
||
"payload"=> Dict(
|
||
"text"=> prompt,
|
||
)
|
||
)
|
||
|
||
attempt = 0
|
||
for attempt in 1:5
|
||
try
|
||
response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
|
||
_responseJsonStr = response["response"]["text"]
|
||
expectedJsonExample =
|
||
"""
|
||
Here is an expected JSON format:
|
||
{
|
||
"text": "...",
|
||
"select": "...",
|
||
"reward": "...",
|
||
"isterminal": "..."
|
||
}
|
||
"""
|
||
responseJsonStr = jsoncorrection(config, _responseJsonStr, expectedJsonExample)
|
||
responseDict = copy(JSON.parsefile(responseJsonStr))
|
||
|
||
text::AbstractString = responseDict["text"]
|
||
select::Union{Nothing, Number} = responseDict["select"] == "null" ? nothing : responseDict["select"]
|
||
reward::Number = responseDict["reward"]
|
||
isterminal::Bool = responseDict["isterminal"]
|
||
|
||
if text != ""
|
||
# pass test
|
||
else
|
||
error("virtual customer not answer correctly")
|
||
end
|
||
|
||
return (text, select, reward, isterminal)
|
||
catch e
|
||
io = IOBuffer()
|
||
showerror(io, e)
|
||
errorMsg = String(take!(io))
|
||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
println("")
|
||
@warn "Error occurred: $errorMsg\n$st"
|
||
println("")
|
||
end
|
||
end
|
||
error("virtualWineUserChatbox failed to get a response")
|
||
end
|
||
|
||
"""
|
||
Search for wines in stock.
|
||
|
||
Executes a wine search via SQL (either through SQLLLM or direct query), optionally fetching
|
||
and base64-encoding bottle images for each result.
|
||
|
||
# Arguments
|
||
- `a::T`: An agent instance (subtype of `agent`) with context containing `executeSQL`,
|
||
`pg_conn_str`, and `agentconfig`
|
||
- `thoughtdict::AbstractDict`: A dictionary containing `action_input` (the search query string)
|
||
|
||
# Keyword Arguments
|
||
- `useSQLLLM::Bool=false`: Whether to use SQLLLM for the query instead of direct SQL generation
|
||
|
||
# Returns
|
||
- `NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}}`: A named tuple with:
|
||
- `thoughtdict`: The input thoughtdict with `action_result` populated
|
||
- `result_raw`: Vector of wine dictionaries with image data, or `nothing`
|
||
|
||
# Notes
|
||
- When `useSQLLLM=false`, performs hard SQL filtering followed by vector search.
|
||
- Fetches bottle images from `http://192.168.88.106:8080/` and encodes as base64.
|
||
- Uses `wine_search_term_classification` to extract search conditions.
|
||
- Requires PostgreSQL connection info from `a.context.agentconfig["externalservice"]["sommpanion_db"]`.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> thoughtdict = OrderedDict("action_input" => "red wine under 50");
|
||
julia> result = YiemAgent.search_wine_database!(agent, thoughtdict)
|
||
(thoughtdict=OrderedDict{String, Any}(...), result_raw=[Dict(...), ...])
|
||
```
|
||
"""
|
||
function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false
|
||
)::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent}
|
||
|
||
println("\ncheckinventory order: $(thoughtdict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
if useSQLLLM
|
||
# add suppport for similarSQLVectorDB
|
||
textresult, result_raw = SQLLLM.query(
|
||
inventoryquery,
|
||
a.context.executeSQL,
|
||
a.context.text2textInstructLLM;
|
||
insertSQLVectorDB=a.context.insertSQLVectorDB,
|
||
similarSQLVectorDB=a.context.similarSQLVectorDB,
|
||
llmFormatName="qwen3")
|
||
thoughtdict["action_result"] = textresult
|
||
else
|
||
|
||
# direct query with possible sql instead of SQLLLM.
|
||
hard_conditions, vector_search = wine_search_term_classification(a, thoughtdict["action_input"])
|
||
|
||
# do hard filter
|
||
# sql = generatesql(a, inventoryquery)
|
||
sql = predefined_wine_search_sql(hard_conditions)
|
||
@info "\nsql: $sql, \nvector_search: $vector_search"
|
||
textresult, sql_result_df, success, _ = SQLexecution(a.context.executeSQL, sql)
|
||
|
||
# do vector search
|
||
vector_search_str = ""
|
||
for i in vector_search
|
||
vector_search_str = vector_search_str * " " * i["value"]
|
||
end
|
||
vector_search_str = String(strip(vector_search_str))
|
||
vector_search_str = GeneralUtils.removestring(vector_search_str, ["%"])
|
||
@show vector_search
|
||
@show vector_search_str
|
||
|
||
|
||
config = a.context.agentconfig
|
||
host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
|
||
port = parse(Int, _port)
|
||
dbname = "winedb"
|
||
user = config["externalservice"]["sommpanion_db"]["user"]
|
||
password = config["externalservice"]["sommpanion_db"]["password"]
|
||
pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password"
|
||
|
||
#WORKING
|
||
# df = GeneralUtils.find_text_vector_similarity(
|
||
# vector_search_str,
|
||
# "wine",
|
||
# "tasting_notes_embedding",
|
||
# GeneralUtils.execute_postgres_sql(pg_conn_str, sql), #BUG input pair (F, arg)
|
||
# a.context.getTextEmbedding([vector_search_str]) #BUG input pair (F, arg)
|
||
# )
|
||
|
||
|
||
# @show df
|
||
# error(888888)
|
||
|
||
|
||
items = nothing
|
||
if sql_result_df !== nothing
|
||
result_vec = GeneralUtils.dfToVectorDict(sql_result_df)
|
||
|
||
# get image
|
||
for d in result_vec
|
||
image_url_json_str = d["image_url"]
|
||
image_url_json_obj = JSON.parse(image_url_json_str)
|
||
base_url = "http://192.168.88.106:8080/"
|
||
if haskey(image_url_json_obj, "bottle")
|
||
url = base_url * image_url_json_obj["bottle"]
|
||
image_data = HTTP.get(url) # vector{int} data
|
||
image_base64_string = base64encode(image_data.body)
|
||
d["image"] = image_base64_string
|
||
else
|
||
d["image"] = nothing
|
||
end
|
||
end
|
||
items = result_vec # image is added to each item
|
||
end
|
||
|
||
thoughtdict["action_result"] = textresult
|
||
end
|
||
|
||
return (thoughtdict=thoughtdict, result_raw=items)
|
||
end
|
||
|
||
|
||
"""
|
||
Generate an SQL query from a natural language search term.
|
||
|
||
Uses an LLM to generate SQL based on the database schema relevant to the search term,
|
||
with validation and retry logic.
|
||
|
||
# Arguments
|
||
- `a::T`: An agent instance (subtype of `agent`) with context containing:
|
||
- `find_related_tables_for_user_question`: Function to find relevant database tables
|
||
- `pg_conn_str`: PostgreSQL connection string
|
||
- `text2textInstructLLM`: LLM function for text generation
|
||
- `searchterm::String`: Natural language search term to convert to SQL
|
||
|
||
# Keyword Arguments
|
||
- `maxattempt::Int=10`: Maximum number of attempts to get a valid SQL response from the LLM
|
||
|
||
# Returns
|
||
- `String`: A valid SQL query string
|
||
|
||
# Notes
|
||
- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model for SQL generation.
|
||
- Dynamically fetches only relevant table schemas for the search term.
|
||
- Validates LLM response contains required keys: "plan", "action_name", "action_input".
|
||
- `action_name` must be "RUNSQL"; `action_input` must not contain "RUNSQL".
|
||
- Strips triple backticks and extracts SQL from fenced code blocks.
|
||
- Throws on failure after `maxattempt` retries.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> sql = YiemAgent.generatesql(agent, "red wine under 50")
|
||
"SELECT ... WHERE w.wine_type = 'red' AND rw.price < 50;"
|
||
```
|
||
"""
|
||
function generatesql(a::T, searchterm::String,
|
||
; maxattempt=10
|
||
)::String where {T<:agent}
|
||
|
||
systemmsg =
|
||
"""
|
||
# database_search_guidelines
|
||
- Keep SQL queries focused only on the provided information.
|
||
- Use wildcard character (%) to search more effectively.
|
||
- Do not create any table in the database.
|
||
- A junction table can be used to link tables together. Another use case is for filtering data.
|
||
- If you can't find a single table that can be used to answer the user's search term, try joining multiple tables to see if you can obtain the answer.
|
||
- Text information in the database usually stored in lower case. If your search returns empty, try using lower case to search.
|
||
- Overly strict condition usually yields empth result
|
||
|
||
# situation
|
||
At each round of conversation, you will be given the following:
|
||
- user search term
|
||
|
||
# objective
|
||
Consult the database_search_guidelines. Then find the data from a database to satisfy the user's search term.
|
||
|
||
# your responsibility includes
|
||
Fulfill the objective.
|
||
|
||
# you should then respond to the user with interleaving plan, action_name, action_input
|
||
1) "plan", Based on the current situation, state a complete action plan to complete the task and rationale. Be specific.
|
||
2) "action_name", Must be "RUNSQL"
|
||
3) "action_input", The input to the action you are about to perform according to your plan.
|
||
After the action is executed you gets "action_result". It is the output from the action you selected.
|
||
|
||
# you should only respond in JSON format as described below
|
||
"plan": "...",
|
||
"action_name": "...",
|
||
"action_input": "..."
|
||
|
||
# available_actions
|
||
"RUNSQL", which you can use to execute SQL against the database.
|
||
The input must be a single SQL query to be executed against the database.
|
||
For more effective text search, it's necessary to use case-insensitivity and the ILIKE operator.
|
||
Do not wrap the SQL as it will be executed against the database directly and SQL must be ended with ';'.
|
||
"""
|
||
|
||
# table_schema =
|
||
# """
|
||
# create table customer (
|
||
# customer_id uuid primary key default gen_random_uuid (),
|
||
# customer_firstname varchar(128),
|
||
# customer_lastname varchar(128),
|
||
# customer_displayname varchar(128) not null,
|
||
# customer_username varchar(128),
|
||
# customer_password varchar(128),
|
||
# customer_gender varchar(128),
|
||
# country varchar(128),
|
||
# telephone varchar(128),
|
||
# email varchar(128) not null,
|
||
# customer_birthdate varchar(128),
|
||
# note text,
|
||
|
||
# other_attributes jsonb,
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp,
|
||
# description text
|
||
# );
|
||
|
||
# create table retailer (
|
||
# retailer_id uuid primary key default gen_random_uuid (),
|
||
# retailer_name varchar(128) not null,
|
||
# retailer_username varchar(128) not null,
|
||
# retailer_password varchar(128) not null,
|
||
# retailer_address text not null,
|
||
# country varchar(128) not null,
|
||
# contact_person varchar(128) not null,
|
||
# telephone varchar(128) not null,
|
||
# email varchar(128) not null,
|
||
# note text,
|
||
|
||
# other_attributes jsonb,
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp,
|
||
# description text
|
||
# );
|
||
|
||
# create table food (
|
||
# food_id uuid primary key default gen_random_uuid (),
|
||
# food_name varchar(128) not null,
|
||
# country varchar(128),
|
||
# spiciness integer,
|
||
# sweetness integer,
|
||
# sourness integer,
|
||
# savoriness integer,
|
||
# bitterness integer,
|
||
# serving_temperature integer,
|
||
# image_url jsonb,
|
||
# note text,
|
||
# other_attributes jsonb,
|
||
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp,
|
||
# description text
|
||
# );
|
||
|
||
# create table wine (
|
||
# wine_id uuid primary key default gen_random_uuid (),
|
||
# seo_name varchar(128) not null,
|
||
# wine_name varchar(128) not null,
|
||
# winery varchar(128) not null,
|
||
# vintage integer not null,
|
||
# region varchar(128) not null,
|
||
# country varchar(128) not null,
|
||
# wine_type varchar(128) not null,
|
||
# grape varchar(128) not null,
|
||
# serving_temperature varchar(128) not null,
|
||
# intensity integer,
|
||
# sweetness integer,
|
||
# tannin integer,
|
||
# acidity integer,
|
||
# fizziness integer,
|
||
# tasting_notes text,
|
||
# image_url jsonb,
|
||
# manufacturer_sku text,
|
||
# note text,
|
||
# other_attributes jsonb,
|
||
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp,
|
||
# description text
|
||
# );
|
||
|
||
# create table wine_food (
|
||
# wine_id uuid references wine(wine_id),
|
||
# food_id uuid references food(food_id),
|
||
# constraint wine_food_id primary key (wine_id, food_id),
|
||
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp
|
||
# );
|
||
|
||
# CREATE TABLE retailer_wine (
|
||
# retailer_id uuid references retailer(retailer_id),
|
||
# wine_id uuid references wine(wine_id),
|
||
# constraint retailer_wine_id primary key (retailer_id, wine_id),
|
||
# price NUMERIC(10, 2),
|
||
# currency varchar(3) not null,
|
||
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp
|
||
# );
|
||
|
||
# CREATE TABLE retailer_food (
|
||
# retailer_id uuid references retailer(retailer_id),
|
||
# food_id uuid references food(food_id),
|
||
# constraint retailer_food_id primary key (retailer_id, food_id),
|
||
# price NUMERIC(10, 2),
|
||
# currency varchar(3) not null,
|
||
|
||
# created_time timestamptz default current_timestamp,
|
||
# updated_time timestamptz default current_timestamp
|
||
# );
|
||
# """
|
||
|
||
requiredKeys = ["plan", "action_name", "action_input"]
|
||
errornote = ""
|
||
# provide similar sql only for the first attempt
|
||
# sql, distance = a.context.similarSQLVectorDB(searchterm)
|
||
|
||
# similarSQL_ = sql !== nothing ? sql : "None"
|
||
# # if sql is really close, just use it
|
||
# if similarSQL_ != "None" && distance <= 0.1
|
||
# return similarSQL_
|
||
# end
|
||
|
||
#CHANGE use find_related_tables_for_user_question and inject only related table schema instead
|
||
# of hard code table schema. CPU embedding is too slow. use embedding service on GPU.
|
||
related_tables = a.context.find_related_tables_for_user_question(searchterm)
|
||
table_schema = ""
|
||
for table in related_tables
|
||
_table_schema_str = GeneralUtils.get_db_table_schema_simple(a.context.pg_conn_str, table)
|
||
table_schema_str = sprint(show, _table_schema_str) * "\n"
|
||
table_schema = table_schema * table_schema_str
|
||
end
|
||
|
||
context =
|
||
"""
|
||
<internal_context_for_assistant>
|
||
<database_table_schema>
|
||
$table_schema
|
||
</database_table_schema>
|
||
</internal_context_for_assistant>
|
||
"""
|
||
input = context * searchterm
|
||
|
||
msg = Dict(
|
||
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||
"messages" => [
|
||
Dict(
|
||
"role" => "system",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => systemmsg),
|
||
]
|
||
),
|
||
Dict(
|
||
"role" => "user",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => input),
|
||
]
|
||
),
|
||
],
|
||
"temperature" => 0.7
|
||
)
|
||
|
||
for attempt in 1:maxattempt
|
||
response = a.context.text2textInstructLLM("random_id", msg)
|
||
|
||
response = GeneralUtils.clean_json_response(response)
|
||
|
||
think, response = GeneralUtils.extractthink(response)
|
||
responsedict = nothing
|
||
try
|
||
_responsedict = JSON.parse(response)
|
||
responsedict = GeneralUtils.dictify(_responsedict, keytype=String, sort_order=requiredKeys)
|
||
catch
|
||
println("\nERROR decisionMaker() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
continue
|
||
end
|
||
|
||
# check whether all answer's key points are in responsedict
|
||
ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
|
||
if !ispass
|
||
errornote = errormsg
|
||
println("\nERROR YiemAgent decisionMaker() $errornote --(not qualify response)> $responsedict ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
|
||
continue
|
||
end
|
||
|
||
# remove backticks Error occurred: MethodError: no method matching occursin(::String, ::Vector{String})
|
||
if occursin("```", responsedict["action_input"])
|
||
sql = GeneralUtils.extract_triple_backtick_text(responsedict["action_input"])[1]
|
||
if sql[1:4] == "sql\n"
|
||
sql = sql[5:end]
|
||
end
|
||
sql = split(sql, ';') # some time there are comments in the sql
|
||
sql = sql[1] * ';'
|
||
|
||
responsedict["action_input"] = sql
|
||
end
|
||
|
||
toollist = ["RUNSQL"]
|
||
if responsedict["action_name"] ∉ toollist
|
||
errornote = "Your previous attempt has action_name that is not in the tool list"
|
||
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["action_name"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
continue
|
||
end
|
||
|
||
for i in toollist
|
||
if occursin(i, responsedict["action_input"])
|
||
errornote = "Your previous attempt has action_name in action_input which is not allowed"
|
||
println("\nERROR SQLLLM decisionMaker(). Attempt $attempt/$maxattempt. $errornote --(not qualify response)--> $(responsedict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
continue
|
||
end
|
||
end
|
||
|
||
# println("\nSQLLLM decisionMaker() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
# pprintln(responsedict)
|
||
# println("---")
|
||
|
||
return responsedict["action_input"]
|
||
end
|
||
error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
|
||
end
|
||
|
||
|
||
"""
|
||
# Example
|
||
```jldoctest
|
||
julia> using ChatAgent
|
||
julia> agent = YiemAgent.sommelier(...)
|
||
julia> thoughtdict =
|
||
OrderedDict{String, Any}(
|
||
"plan" => "The user is asking a very specific question about a wine (Brunello di Montalcino from Tenuta CastelGiocondo). Although the policy suggests gathering budget, wine type, and occasion, the user has provided enough specific information (name, region, producer) to attempt a direct search in the database. I will use the SEARCH_WINE_DATABASE action to check if this specific wine is in our inventory.",
|
||
"action_name" => "SEARCH_WINE_DATABASE",
|
||
"action_input" => "Brunello di Montalcino from Tenuta CastelGiocondo")
|
||
```
|
||
julia> predefined_wine_search_sql(agent, thoughtdict["action_input"])
|
||
"""
|
||
"""
|
||
Classify a wine search term into hard SQL conditions and vector search words.
|
||
|
||
Uses an LLM to extract database column conditions from a natural language search term,
|
||
then classifies them into hard conditions (for SQL WHERE clauses) and vector search
|
||
entries (for approximate matching).
|
||
|
||
# Arguments
|
||
- `a::T`: An agent instance (subtype of `agent`) with context containing:
|
||
- `find_related_tables_for_user_question`: Function to find relevant tables
|
||
- `pg_conn_str`: PostgreSQL connection string
|
||
- `text2textInstructLLM`: LLM function for text generation
|
||
- `searchterm::String`: Natural language search term to classify
|
||
|
||
# Keyword Arguments
|
||
- `maxattempt::Int=10`: Maximum number of attempts to get a valid classification from the LLM
|
||
|
||
# Returns
|
||
- `NamedTuple{(:hard_conditions, :vector_search), Tuple{Vector{JSON.Object{String, Any}}, Vector{JSON.Object{String, Any}}}}`:
|
||
- `hard_conditions`: Entries with standard operators (=, <>, !=, >, <, >=, <=) for SQL WHERE clauses
|
||
- `vector_search`: Entries with non-standard operators (LIKE, IN, IS NULL, etc.) for vector search
|
||
|
||
# Notes
|
||
- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model with JSON schema response format.
|
||
- Applies fuzzy correction for "fuzzy_correction" bucket columns via `resolve_entity`.
|
||
- Uses `classify_column` to determine the column type bucket.
|
||
- Uses `harvest_entity_catalog` and `resolve_entity` (threshold=0.9) for fuzzy matching.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> result = YiemAgent.wine_search_term_classification(agent, "dry red wine under 30")
|
||
(hard_conditions=[...], vector_search=[...])
|
||
```
|
||
"""
|
||
function wine_search_term_classification(a::T, searchterm::String,
|
||
; maxattempt=10
|
||
) where {T<:agent}
|
||
|
||
systemmsg =
|
||
"""
|
||
|
||
# situation
|
||
At each round of conversation, you will be given the following:
|
||
- user search term
|
||
- database tables schema
|
||
|
||
# objective
|
||
Consult the provided database schema (tables and columns), please map a user's natural-language search term to the appropriate database columns and tables—identify the relevant fields, operators, and values (e.g., for SQL filtering).
|
||
|
||
# your responsibility includes
|
||
Fulfill the objective.
|
||
|
||
# You must output your response as a JSON object containing a single key: "extracted_info".
|
||
The "extracted_info" key must contain an array of objects. Each object must contain:
|
||
1) "table_name": The name of the table.
|
||
2) "column_name": The specific column being filtered.
|
||
3) "operator": The comparison operator (e.g., "=", ">").
|
||
4) "value": The value to compare against.
|
||
|
||
If the user does not specify any filters, return an empty array for "extracted_info": {"extracted_info": []}.
|
||
|
||
# here are some example
|
||
<user>
|
||
4-wheel drive car with red color that will give me fast and furious emotion. No more than 7000 USD
|
||
</user>
|
||
<assistant>
|
||
{
|
||
"extracted_info": [
|
||
{
|
||
"table_name": "car_info",
|
||
"column_name": "drive_type",
|
||
"operator": "=",
|
||
"value": "4-wheel"
|
||
},
|
||
{
|
||
"table_name": "car_info",
|
||
"column_name": "color",
|
||
"operator": "=",
|
||
"value": "red"
|
||
},
|
||
{
|
||
"table_name": "car_info",
|
||
"column_name": "drive_feeling",
|
||
"operator": "=",
|
||
"value": "fast and furious"
|
||
},
|
||
{
|
||
"table_name": "price_list",
|
||
"column_name": "price",
|
||
"operator": "<",
|
||
"value": "7000"
|
||
}
|
||
}
|
||
</assistant>
|
||
"""
|
||
|
||
|
||
# use find_related_tables_for_user_question and inject only related table schema for a given search term
|
||
# to LLM instead of giving LLM all tables schema.
|
||
related_tables = a.context.find_related_tables_for_user_question(searchterm)
|
||
table_schema = ""
|
||
for table in related_tables
|
||
_table_schema_str = get_db_table_schema_simple_with_samples(a.context.pg_conn_str, table)
|
||
|
||
# _table_schema_str = GeneralUtils.get_db_table_schema_simple(a.context.pg_conn_str, table)
|
||
table_schema_str = sprint(show, _table_schema_str) * "\n"
|
||
table_schema = table_schema * table_schema_str
|
||
end
|
||
|
||
context =
|
||
"""
|
||
<internal_context_for_assistant>
|
||
<database_table_schema>
|
||
$table_schema
|
||
</database_table_schema>
|
||
</internal_context_for_assistant>
|
||
"""
|
||
input = context * searchterm
|
||
|
||
response_format = Dict(
|
||
"type" => "json_schema",
|
||
"json_schema" => Dict(
|
||
"name" => "extracted_conditions",
|
||
"strict" => true,
|
||
"schema" => Dict(
|
||
"type" => "object",
|
||
"properties" => Dict(
|
||
"extracted_info" => Dict(
|
||
"type" => "array",
|
||
"items" => Dict(
|
||
"type" => "object",
|
||
"properties" => Dict(
|
||
"table_name" => Dict(
|
||
"type" => "string",
|
||
"description" => "The name of the database table."
|
||
),
|
||
"column_name" => Dict(
|
||
"type" => "string",
|
||
"description" => "The name of the column to filter on."
|
||
),
|
||
"operator" => Dict(
|
||
"type" => "string",
|
||
"enum" => ["=", "!=", ">", "<", ">=", "<=", "LIKE", "IN", "IS NULL", "IS NOT NULL"],
|
||
"description" => "The SQL comparison operator."
|
||
),
|
||
"value" => Dict(
|
||
"type" => ["string", "null"],
|
||
"description" => "The value to compare against. Use null for IS NULL/IS NOT NULL."
|
||
)
|
||
),
|
||
"required" => ["table_name", "column_name", "operator", "value"],
|
||
"additionalProperties" => false
|
||
)
|
||
)
|
||
),
|
||
"required" => ["extracted_info"],
|
||
"additionalProperties" => false
|
||
)
|
||
)
|
||
)
|
||
|
||
msg = Dict(
|
||
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||
"messages" => [
|
||
Dict(
|
||
"role" => "system",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => systemmsg),
|
||
]
|
||
),
|
||
Dict(
|
||
"role" => "user",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => input),
|
||
]
|
||
),
|
||
],
|
||
"temperature" => 0.7,
|
||
"response_format"=> response_format,
|
||
)
|
||
|
||
for attempt in 1:maxattempt
|
||
response = a.context.text2textInstructLLM("random_id", msg)
|
||
responsedict = JSON.parse(response)
|
||
# responsedict = nothing
|
||
# try
|
||
# responsedict = Serde.parse_yaml(response)
|
||
# catch e
|
||
# println("\nERROR YiemAgent predefined_wine_search_sql() Error: $e --(not qualify response)-> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
|
||
# continue
|
||
# end
|
||
|
||
# println("\n ", table_schema)
|
||
println("\n ", responsedict)
|
||
@info "before BM25 " @__LINE__
|
||
|
||
# to ensure user input is correct
|
||
for entry in responsedict["extracted_info"]
|
||
table_name = entry["table_name"]::String
|
||
column_name = entry["column_name"]::String
|
||
|
||
bucket = classify_column(a.context.pg_conn_str, table_name, column_name)
|
||
|
||
if bucket == "fuzzy_correction"
|
||
words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, table_name, column_name)
|
||
resolved_word = GeneralUtils.resolve_entity(entry["value"], words_catalog; threshold=0.9)
|
||
entry["value"] = resolved_word
|
||
end
|
||
end
|
||
|
||
# filter for column that will be used for hard condition (SQL where clause)
|
||
# column with non-standard operator will be used in vector search
|
||
vector_search_words = ""
|
||
hard_operators = ["=","<>","!=",">","<",">=","<=","!<","!>","<=>"]
|
||
|
||
# Build new list of hard condition entries
|
||
hard_conditions = JSON.Object{String, Any}[]
|
||
vector_search = JSON.Object{String, Any}[]
|
||
for entry in responsedict["extracted_info"]
|
||
if entry["operator"] ∈ hard_operators
|
||
push!(hard_conditions, entry)
|
||
else
|
||
push!(vector_search, entry)
|
||
end
|
||
end
|
||
responsedict = hard_conditions
|
||
|
||
println("")
|
||
@show responsedict
|
||
@info "predefined_wine_search_sql() " @__LINE__
|
||
|
||
return (hard_conditions=hard_conditions, vector_search=vector_search)
|
||
end
|
||
error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
|
||
end
|
||
|
||
"""
|
||
Build a SQL query from pre-classified wine search conditions.
|
||
|
||
Constructs a JOIN query across `wine`, `retailer_wine`, and `retailer` tables with
|
||
dynamic WHERE clauses based on the provided conditions.
|
||
|
||
# Arguments
|
||
- `conditions::Vector{JSON.Object{String, Any}}`: Vector of condition objects, each containing:
|
||
- `table_name`: One of "wine", "retailer_wine" (other tables are skipped)
|
||
- `column_name`: The column to filter on
|
||
- `operator`: SQL comparison operator (=, <>, !=, >, <, >=, <=)
|
||
- `value`: The value to compare against (number or string)
|
||
|
||
# Returns
|
||
- `String`: A complete SQL query with WHERE clause
|
||
|
||
# Notes
|
||
- Supports table aliases: "wine" → "w", "retailer_wine" → "rw"
|
||
- Automatically handles numeric vs string value types in SQL formatting
|
||
- Strings are single-quote escaped (replaces "'" with "''")
|
||
- Returns a query with no WHERE clause if conditions vector is empty
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> cond = [JSON.Object{String, Any}("table_name"=>"wine", "column_name"=>"wine_type", "operator"=>"=", "value"=>"red")];
|
||
julia> YiemAgent.predefined_wine_search_sql(cond)
|
||
"SELECT ... FROM wine AS w JOIN ... WHERE w.wine_type = 'red';"
|
||
```
|
||
"""
|
||
function predefined_wine_search_sql(conditions::Vector{JSON.Object{String, Any}})::String
|
||
# 1. Base SQL structure
|
||
base_query =
|
||
"""
|
||
SELECT
|
||
w.winery,
|
||
w.wine_name,
|
||
w.wine_id,
|
||
w.vintage,
|
||
w.region,
|
||
w.country,
|
||
w.wine_type,
|
||
w.grape,
|
||
w.serving_temperature,
|
||
w.sweetness,
|
||
w.intensity,
|
||
w.tannin,
|
||
w.acidity,
|
||
w.tasting_notes,
|
||
rw.price,
|
||
rw.currency,
|
||
w.image_url,
|
||
r.retailer_name,
|
||
rw.retailer_id
|
||
FROM wine AS w
|
||
JOIN retailer_wine AS rw ON w.wine_id = rw.wine_id
|
||
JOIN retailer AS r ON rw.retailer_id = r.retailer_id
|
||
"""
|
||
|
||
# 2. Dynamic WHERE Clause Builder
|
||
where_clauses = String[]
|
||
|
||
# Iterate over each condition object in the array
|
||
for cond in conditions
|
||
table_name = String(cond["table_name"])
|
||
column_name = String(cond["column_name"])
|
||
op = String(cond["operator"])
|
||
raw_val = cond["value"]
|
||
|
||
# Determine table alias
|
||
alias = if table_name == "wine"
|
||
"w"
|
||
elseif table_name == "retailer_wine"
|
||
"rw"
|
||
else
|
||
continue
|
||
end
|
||
|
||
# --- Value Type Handling ---
|
||
final_val = raw_val
|
||
|
||
if op in ("=", "<", ">", "<=", ">=")
|
||
str_val = string(raw_val)
|
||
num_val = tryparse(Float64, str_val)
|
||
|
||
if !isnothing(num_val)
|
||
final_val = isinteger(num_val) ? round(Int, num_val) : num_val
|
||
end
|
||
end
|
||
|
||
# --- SQL Formatting ---
|
||
if isa(final_val, Number)
|
||
clause = "$(alias).$(column_name) $(op) $(final_val)"
|
||
else
|
||
escaped_val = replace(string(final_val), "'" => "''")
|
||
clause = "$(alias).$(column_name) $(op) '$(escaped_val)'"
|
||
end
|
||
|
||
push!(where_clauses, clause)
|
||
end
|
||
|
||
# 3. Assemble Final Query
|
||
where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ")
|
||
|
||
return string(base_query, where_sql, ";")
|
||
end
|
||
|
||
"""
|
||
Execute a SQL query against the database and return formatted results.
|
||
|
||
Adds `ORDER BY RANDOM() LIMIT 2` for non-LIMITed queries, removes `DISTINCT`, and returns
|
||
either a formatted string result or the DataFrame.
|
||
|
||
# Arguments
|
||
- `executeSQL::Function`: A function that executes SQL and returns results (e.g., PostgreSQL connection)
|
||
- `sql::T`: The SQL query string to execute (subtype of `AbstractString`)
|
||
|
||
# Returns
|
||
- `NamedTuple{(:result_str, :result_raw, :success, :errormsg)}`: A named tuple with:
|
||
- `result_str::Union{String, Nothing}`: Formatted string representation of results
|
||
- `result_raw::Union{DataFrame, Nothing}`: The DataFrame result, or `nothing`
|
||
- `success::Bool`: Whether the query executed successfully
|
||
- `errormsg::Union{String, Nothing}`: Error message if failed, or `nothing`
|
||
|
||
# Notes
|
||
- Removes `DISTINCT` keyword before execution (incompatible with `RANDOM()`)
|
||
- Appends `ORDER BY RANDOM() LIMIT 2` if query doesn't have `LIMIT` and ends with `;`
|
||
- Returns "No records found" message if zero rows
|
||
- Returns column count warning if more than 30 columns
|
||
- Randomly samples 2 rows if result has more than 2 rows
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> result = YiemAgent.SQLexecution(execute_fn, "SELECT * FROM wine;")
|
||
(result_str="...", result_raw=DataFrame(...), success=true, errormsg=nothing)
|
||
```
|
||
"""
|
||
function SQLexecution(executeSQL::Function, sql::T
|
||
)::NamedTuple where {T<:AbstractString}
|
||
|
||
try
|
||
# add LIMIT to the SQL to prevent loading large data
|
||
sql = strip(sql)
|
||
|
||
# remove DISTINCT keyword because it is incompatible with RANDOM()
|
||
sql = replace(sql, "DISTINCT" => "")
|
||
|
||
if sql[end] == ';'
|
||
if !occursin("LIMIT", sql)
|
||
sql = sql[1:end-1] * " ORDER BY RANDOM() LIMIT 2;"
|
||
end
|
||
else
|
||
sql = sql * ";"
|
||
end
|
||
result = executeSQL(sql)
|
||
df = DataFrame(result)
|
||
tablesize = size(df)
|
||
row, column = tablesize
|
||
if row == 0
|
||
return (result_str="No records found. Try loosening your search criteria.", result_raw=nothing, success=true, errormsg=nothing)
|
||
elseif column > 30
|
||
return (result_str="There are more than 30 columns. Please be more specific.", result_raw=df, success=true, errormsg=nothing)
|
||
else
|
||
df1 =
|
||
if row > 2
|
||
# ramdom row to pick
|
||
df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df
|
||
else
|
||
df
|
||
end
|
||
result = GeneralUtils.dfToString(df1)
|
||
# println("\n~~~ SQLexecution() result: ", @__FILE__, " ", @__LINE__)
|
||
# println(sql)
|
||
# println(df1)
|
||
# println("\n")
|
||
return (result_str=result, result_raw=df1, success=true, errormsg=nothing)
|
||
end
|
||
catch e
|
||
io = IOBuffer()
|
||
showerror(io, e)
|
||
errorMsg = String(take!(io))
|
||
st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace()))
|
||
println(errorMsg)
|
||
return (result_str=nothing, result_raw=nothing, success=false, errormsg=errorMsg)
|
||
end
|
||
end
|
||
|
||
"""
|
||
DEPRECATED: Search for wines in stock (legacy implementation).
|
||
|
||
Use `search_wine_database!` instead. This function uses the older approach with
|
||
`extractWineAttributes_1` and `extractWineAttributes_2` for attribute extraction.
|
||
|
||
# Arguments
|
||
- `a::T`: An agent instance (subtype of `agent`)
|
||
- `thoughtdict::AbstractDict`: Dictionary containing `action_input` (search query)
|
||
|
||
# Keyword Arguments
|
||
- `useSQLLLM::Bool=false`: Whether to use SQLLLM for the query
|
||
|
||
# Returns
|
||
- `NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}}`
|
||
|
||
# Notes
|
||
- DEPRECATED: Use `search_wine_database!` for the current implementation.
|
||
- Calls `extractWineAttributes_1` and `extractWineAttributes_2` for attribute extraction.
|
||
"""
|
||
function DEPRECIATED_search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false
|
||
)::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent}
|
||
|
||
# XXX
|
||
predefined_wine_search_sql(a, thoughtdict["action_input"])
|
||
|
||
println("\ncheckinventory order: $(thoughtdict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
wineattributes_1 = extractWineAttributes_1(a, thoughtdict["action_input"])
|
||
wineattributes_2 = extractWineAttributes_2(a, thoughtdict["action_input"])
|
||
|
||
retrieve_attributes = ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"]
|
||
_inventoryquery = "$(thoughtdict["action_input"]), $wineattributes_1, $wineattributes_2, retailer_name: $(a.retailername), retailerid: $(a.retailerid)"
|
||
inventoryquery = "Retrieves $retrieve_attributes of wines that match the following criteria - {$_inventoryquery}"
|
||
println("\ncheckinventory input: $inventoryquery ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
|
||
if useSQLLLM
|
||
# add suppport for similarSQLVectorDB
|
||
textresult, result_raw = SQLLLM.query(
|
||
inventoryquery,
|
||
a.context.executeSQL,
|
||
a.context.text2textInstructLLM;
|
||
insertSQLVectorDB=a.context.insertSQLVectorDB,
|
||
similarSQLVectorDB=a.context.similarSQLVectorDB,
|
||
llmFormatName="qwen3")
|
||
thoughtdict["action_result"] = textresult
|
||
else
|
||
|
||
# direct query with possible sql instead of SQLLLM.
|
||
sql = generatesql(a, inventoryquery)
|
||
println("\nSQL: $sql ", @__FILE__, ":", @__LINE__, " $(Dates.now()) \n")
|
||
textresult, sql_result_df, success, _ = SQLexecution(a.context.executeSQL, sql)
|
||
|
||
items = nothing
|
||
if sql_result_df !== nothing
|
||
result_vec = GeneralUtils.dfToVectorDict(sql_result_df)
|
||
|
||
# get image
|
||
for d in result_vec
|
||
image_url_json_str = d["image_url"]
|
||
image_url_json_obj = JSON.parse(image_url_json_str)
|
||
base_url = "http://192.168.88.106:8080/"
|
||
if haskey(image_url_json_obj, "bottle")
|
||
url = base_url * image_url_json_obj["bottle"]
|
||
image_data = HTTP.get(url) # vector{int} data
|
||
image_base64_string = base64encode(image_data.body)
|
||
d["image"] = image_base64_string
|
||
else
|
||
d["image"] = nothing
|
||
end
|
||
end
|
||
items = result_vec # image is added to each item
|
||
end
|
||
|
||
thoughtdict["action_result"] = textresult
|
||
end
|
||
|
||
return (thoughtdict=thoughtdict, result_raw=items)
|
||
end
|
||
|
||
"""
|
||
Extract wine attributes from a user's search query.
|
||
|
||
Uses an LLM to parse natural language input into structured wine attributes including
|
||
name, winery, vintage, country, type, grape, price range, occasion, and food pairing.
|
||
|
||
# Arguments
|
||
- `a::T1`: An agent instance (subtype of `agent`) with context containing:
|
||
- `text2textInstructLLM`: LLM function for text generation
|
||
- `pg_conn_str`: PostgreSQL connection string
|
||
- `id`: Agent identifier
|
||
- `input::T2`: User's search query string (subtype of `AbstractString`)
|
||
|
||
# Keyword Arguments
|
||
- `maxattempt::Int=10`: Maximum number of attempts to get a valid response from the LLM
|
||
|
||
# Returns
|
||
- `String`: Comma-separated list of extracted attributes in the format `"key: value, key: value"`
|
||
Attributes with "N/A", empty, or "none" values are excluded.
|
||
|
||
# Notes
|
||
- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model with temperature 0.7.
|
||
- Extracts: wine_name, winery, vintage, country, wine_type, grape_varietal, tasting_notes,
|
||
wine_price_min, wine_price_max, occasion, food_to_be_paired_with_wine
|
||
- Validates response contains all required keys via `checkAgentResponse_JSON`.
|
||
- Applies fuzzy entity resolution via `harvest_entity_catalog` and `resolve_entity` (threshold=0.9).
|
||
- Strips "(some comment)" patterns from values.
|
||
- Removes keys: thought, tasting_notes, occasion, food_to_be_paired_with_wine, vintage from final output.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> YiemAgent.extractWineAttributes_1(agent, "red wine from Napa under 50")
|
||
"wine_name:N/A, winery:N/A, vintage:N/A, country:United States, wine_type:red, grape_varietal:N/A, wine_price_min:0, wine_price_max:50"
|
||
```
|
||
"""
|
||
function extractWineAttributes_1(a::T1, input::T2; maxattempt=10
|
||
)::String where {T1<:agent, T2<:AbstractString}
|
||
|
||
systemmsg =
|
||
"""
|
||
<situation>
|
||
At each round of conversation, the user provides the following:
|
||
- The query: the query provided by the user.
|
||
</situation>
|
||
<objective>
|
||
Extract information from the user's query as much as possible according to wine attributes extraction guidelines to fill out user's preference form.
|
||
</objective>
|
||
<your responsibility includes>
|
||
Fulfill the objective.
|
||
</your responsibility includes>
|
||
<wine attributes extraction guidelines>
|
||
- If specific information required in the preference form is not available in the query or there isn't any, mark with "N/A" to indicate this.
|
||
Additionally, words like 'any' or 'unlimited' mean no information is available.
|
||
- Do not generate other comments.
|
||
</wine attributes extraction guidelines>
|
||
<you should then respond to the user with>
|
||
wine_name: name of the wine
|
||
winery: name of the winery
|
||
vintage: the year of the wine
|
||
country: a country where wine is produced. Can be "Austria", "Australia", "France", "Germany", "Italy", "Portugal", "Spain", "United States". Use "or" if there are multiple countries.
|
||
wine_type: can be one of: "red", "white", "sparkling", "rose", "dessert" or "fortified"
|
||
grape_varietal: the name of the primary grape used to make the wine
|
||
tasting_notes: a word describe the wine's flavor, such as "butter", "oak", "fruity", "raspberry", "earthy", "floral", etc
|
||
wine_price_min: minimum price range of wine. Example: For wine price 20, wine_price_min will be 0. For wine price 10 to 100, wine_price_min will be 10.
|
||
wine_price_max: maximum price range of wine. Example: For wine price 20, wine_price_max will be 20. For wine price 10 to 100, wine_price_max will be 100.
|
||
occasion: the occasion the user is having the wine for
|
||
food_to_be_paired_with_wine: food that the user will be served with the wine such as poultry, fish, steak, etc
|
||
_keyword suffice is the related keyword that appears in user's query.
|
||
</you should then respond to the user with>
|
||
<you should only respond in JSON format as described below>
|
||
"wine_name": "...",
|
||
"winery": "...",
|
||
"vintage": "...",
|
||
"country": "...",
|
||
"wine_type": "...",
|
||
"grape_varietal": "...",
|
||
"tasting_notes": "...",
|
||
"wine_price_min": "...",
|
||
"wine_price_max": "...",
|
||
"occasion": "...",
|
||
"food_to_be_paired_with_wine": "..."
|
||
</you should only respond in JSON format as described below>
|
||
<here are some examples>
|
||
User's query: red, Chenin Blanc, Riesling, 20 USD from Tuscany, Italy or Napa Valley, USA
|
||
"wine_name": "N/A",
|
||
"winery": "N/A",
|
||
"vintage": "N/A",
|
||
"country": "Italy or United States",
|
||
"wine_type": "red or white",
|
||
"grape_varietal": "Chenin Blanc or Riesling",
|
||
"tasting_notes": "citrus",
|
||
"wine_price_min": "0",
|
||
"wine_price_max": "20",
|
||
"occasion": "N/A",
|
||
"food_to_be_paired_with_wine": "N/A"
|
||
|
||
User's query: Domaine du Collier Saumur Blanc 2019, France, white, Merlot
|
||
"wine_name": "Saumur Blanc",
|
||
"winery": "Domaine du Collier",
|
||
"vintage": "2019",
|
||
"country": "France",
|
||
"wine_type": "white",
|
||
"grape_varietal": "Merlot",
|
||
"tasting_notes": "N/A",
|
||
"wine_price_min": "N/A",
|
||
"wine_price_max": "N/A",
|
||
"occasion": "N/A",
|
||
"food_to_be_paired_with_wine": "N/A"
|
||
</here are some examples>
|
||
"""
|
||
requiredKeys = ["wine_name", "winery", "vintage", "country", "wine_type", "grape_varietal", "tasting_notes", "wine_price_min", "wine_price_max", "occasion", "food_to_be_paired_with_wine"]
|
||
errornote = ""
|
||
context =
|
||
"""
|
||
<internal_context_for_assistant>
|
||
$errornote
|
||
</internal_context_for_assistant>
|
||
"""
|
||
|
||
input = context * input
|
||
|
||
msg = Dict(
|
||
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||
"messages" => [
|
||
Dict(
|
||
"role" => "system",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => systemmsg),
|
||
]
|
||
),
|
||
Dict(
|
||
"role" => "user",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => input),
|
||
]
|
||
),
|
||
],
|
||
"temperature" => 0.7
|
||
)
|
||
|
||
for attempt in 1:maxattempt
|
||
response = a.context.text2textInstructLLM(a.id, msg)
|
||
response = GeneralUtils.clean_json_response(response)
|
||
|
||
response = GeneralUtils.remove_french_accents(response)
|
||
think, response = GeneralUtils.extractthink(response)
|
||
responsedict = nothing
|
||
try
|
||
_responsedict = JSON.parse(response)
|
||
responsedict = GeneralUtils.dictify(_responsedict; keytype=String, sort_order=requiredKeys)
|
||
catch
|
||
println("\nERROR YiemAgent extractWineAttributes_1() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
continue
|
||
end
|
||
# check whether all answer's key points are in responsedict
|
||
ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
|
||
if !ispass
|
||
errornote = errormsg
|
||
println("\nERROR YiemAgent extractWineAttributes_1() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
|
||
continue
|
||
end
|
||
removekeys = ["thought", "tasting_notes", "occasion", "food_to_be_paired_with_wine", "vintage"]
|
||
for i in removekeys
|
||
delete!(responsedict, i)
|
||
end
|
||
# remove (some text)
|
||
for (k, v) in responsedict
|
||
_v = replace(v, r"\(.*?\)" => "")
|
||
responsedict[k] = _v
|
||
end
|
||
|
||
@info "YiemAgent extractWineAttributes_1() " @__LINE__
|
||
@show responsedict
|
||
@info "---\n" @__LINE__
|
||
|
||
# check each attributes against each column in a database table with BM25
|
||
for (k, v) in responsedict
|
||
if k ∉ ["wine_price_min", "wine_price_max"]
|
||
words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, "wine", k)
|
||
resolved_word = GeneralUtils.resolve_entity(v, words_catalog; threshold=0.9)
|
||
responsedict[k] = resolved_word
|
||
end
|
||
end
|
||
|
||
result = ""
|
||
for (k, v) in responsedict
|
||
# some time LLM generate text with "(some comment)". this line removes it
|
||
if !occursin("N/A", v) && v != "" && !occursin("none", v) && !occursin("None", v)
|
||
result *= "$k: $v, "
|
||
end
|
||
end
|
||
|
||
result = result[1:end-2] # remove the ending ", "
|
||
|
||
@info "YiemAgent extractWineAttributes_1() " @__LINE__
|
||
@show result
|
||
@info "---\n" @__LINE__
|
||
|
||
return result
|
||
end
|
||
error("extractWineAttributes_1() failed to get a response")
|
||
end
|
||
|
||
"""
|
||
Extract wine intensity, sweetness, tannin, and acidity attributes from a query.
|
||
|
||
Uses an LLM with a conversion table to map descriptive words (e.g., "medium-bodied", "low acidity")
|
||
to integer ranges on a 1-5 scale.
|
||
|
||
# Arguments
|
||
- `a::T1`: An agent instance (subtype of `agent`) with context containing:
|
||
- `text2textInstructLLM`: LLM function for text generation
|
||
- `id`: Agent identifier
|
||
- `input::T2`: User's query string containing descriptive wine preferences
|
||
(subtype of `AbstractString`)
|
||
|
||
# Returns
|
||
- `String`: Comma-separated list of extracted attributes in the format
|
||
`"key: value, key: value"`. Only includes numeric values or non-N/A strings.
|
||
|
||
# Notes
|
||
- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model with temperature 0.7.
|
||
- Extracts: sweetness, acidity, tannin, intensity (each with min/max values).
|
||
- Applies `remove_french_accents` to the LLM response.
|
||
- Extracts thinking via `extractthink` before JSON parsing.
|
||
- Validates response contains all required keys via `checkAgentResponse_JSON`.
|
||
- Removes keyword fields (sweetness_keyword, acidity_keyword, etc.) from final output.
|
||
- Only includes values that are numbers or non-"N/A" strings.
|
||
|
||
# TODO
|
||
- "French dry white wines with medium bod" — the LLM does not recognize sweetness. Use LLM self-questioning to solve.
|
||
- French Syrah, Viognier, under 100 — LLM extracts intensity of 3-5. Investigate why.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> YiemAgent.extractWineAttributes_2(agent, "medium-bodied, low acidity, medium tannin")
|
||
"acidity_min:1, acidity_max:2, tannin_min:3, tannin_max:4, intensity_min:3, intensity_max:4"
|
||
```
|
||
"""
|
||
function extractWineAttributes_2(a::T1, input::T2)::String where {T1<:agent, T2<:AbstractString}
|
||
|
||
conversiontable =
|
||
"""
|
||
<conversion_table>
|
||
Intensity level:
|
||
1 to 2: May correspond to "light-bodied" or a similar description.
|
||
2 to 3: May correspond to "med light bodied", "medium light" or a similar description.
|
||
3 to 4: May correspond to "medium bodied" or a similar description.
|
||
4 to 5: May correspond to "med full bodied", "medium full" or a similar description.
|
||
4 to 5: May correspond to "full bodied" or a similar description.
|
||
Sweetness level:
|
||
1 to 2: May correspond to "dry", "no sweet" or a similar description.
|
||
2 to 3: May correspond to "off dry", "less sweet" or a similar description.
|
||
3 to 4: May correspond to "semi sweet" or a similar description.
|
||
4 to 5: May correspond to "sweet" or a similar description.
|
||
4 to 5: May correspond to "very sweet" or a similar description.
|
||
Tannin level:
|
||
1 to 2: May correspond to "low tannin" or a similar description.
|
||
2 to 3: May correspond to "semi low tannin" or a similar description.
|
||
3 to 4: May correspond to "medium tannin" or a similar description.
|
||
4 to 5: May correspond to "semi high tannin" or a similar description.
|
||
4 to 5: May correspond to "high tannin" or a similar description.
|
||
Acidity level:
|
||
1 to 2: May correspond to "low acidity" or a similar description.
|
||
2 to 3: May correspond to "semi low acidity" or a similar description.
|
||
3 to 4: May correspond to "medium acidity" or a similar description.
|
||
4 to 5: May correspond to "semi high acidity" or a similar description.
|
||
4 to 5: May correspond to "high acidity" or a similar description.
|
||
</conversion_table>
|
||
"""
|
||
|
||
systemmsg =
|
||
"""
|
||
<situation>
|
||
At each round of conversation, you will be given the following information:
|
||
conversion_table: a conversion table that maps descriptive words to their corresponding integer levels
|
||
query: the words from the user's query that describe their preferences
|
||
</situation>
|
||
<objective>
|
||
Fill out the user's preference form based on the corresponding words from the user's query according to the guidelines.
|
||
</objective>
|
||
<your responsibility includes>
|
||
Fulfill the objective
|
||
</your responsibility includes>
|
||
<guidelines>
|
||
- The preference form requires sweetness, acidity, tannin, intensity infomation
|
||
- If specific information required in the preference form is not available in the query or there isn't any, mark with 'N/A' to indicate this.
|
||
Additionally, words like 'any' or 'unlimited' mean no information is available.
|
||
- Use the conversion table to convert the descriptive word level of sweetness, intensity, tannin, and acidity into a corresponding integer.
|
||
- Do not generate other comments.
|
||
</guidelines>
|
||
<you should then respond to the user with>
|
||
sweetness_keyword: The exact keywords in the user's query describing the sweetness level of the wine.
|
||
sweetness: ( S ), where ( S ) represents integers indicating the range of sweetness levels. Example: 1-2
|
||
acidity_keyword: The exact keywords in the user's query describing the acidity level of the wine.
|
||
acidity: ( A ), where ( A ) represents integers indicating the range of acidity level. Example: 3-5
|
||
tannin_keyword: The exact keywords in the user's query describing the tannin level of the wine.
|
||
tannin: ( T ), where ( T ) represents integers indicating the range of tannin level. Example: 1-3
|
||
intensity_keyword: The exact keywords in the user's query describing the intensity level of the wine.
|
||
intensity: ( I ), where ( I ) represents integers indicating the range of intensity level. Example: 2-4
|
||
</you should then respond to the user with>
|
||
<you should only respond in JSON format as described below>
|
||
"sweetness_keyword": "...",
|
||
"sweetness_min": "...",
|
||
"sweetness_max": "...",
|
||
"acidity_keyword": "...",
|
||
"acidity_min": "...",
|
||
"acidity_max": "...",
|
||
"tannin_keyword": "...",
|
||
"tannin_min": "...",
|
||
"tannin_max": "...",
|
||
"intensity_keyword": "...",
|
||
"intensity_min": "...",
|
||
"intensity_max": "..."
|
||
</you should only respond in JSON format as described below>
|
||
<here are some examples>
|
||
User's query: I want a wine with a medium-bodied, low acidity, medium tannin.
|
||
"sweetness_keyword": "N/A",
|
||
"sweetness_min": "N/A",
|
||
"sweetness_max": "N/A",
|
||
"acidity_keyword": "low acidity",
|
||
"acidity_min": 1,
|
||
"acidity_max": 2,
|
||
"tannin_keyword": "medium tannin",
|
||
"tannin_min": 3,
|
||
"tannin_max": 4,
|
||
"intensity_keyword": "medium-bodied",
|
||
"intensity_min": 3,
|
||
"intensity_max": 4
|
||
|
||
User's query: German red wine, under 100, pairs with spicy food.
|
||
"sweetness_keyword": "N/A",
|
||
"sweetness_min": "N/A",
|
||
"sweetness_max": "N/A",
|
||
"acidity_keyword": "N/A",
|
||
"acidity_min": "N/A",
|
||
"acidity_max": "N/A",
|
||
"tannin_keyword": "N/A",
|
||
"tannin_min": "N/A",
|
||
"tannin_max": "N/A",
|
||
"intensity_keyword": "N/A",
|
||
"intensity_min": "N/A",
|
||
"intensity_max": "N/A"
|
||
<here are some examples>
|
||
"""
|
||
requiredKeys = ["sweetness_keyword", "sweetness_min", "sweetness_max",
|
||
"acidity_keyword", "acidity_min", "acidity_max",
|
||
"tannin_keyword", "tannin_min", "tannin_max",
|
||
"intensity_keyword", "intensity_min", "intensity_max"]
|
||
errornote = ""
|
||
context =
|
||
"""
|
||
<internal_context_for_assistant>
|
||
$conversiontable
|
||
$errornote
|
||
</internal_context_for_assistant>
|
||
"""
|
||
|
||
input = context * input
|
||
|
||
msg = Dict(
|
||
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||
"messages" => [
|
||
Dict(
|
||
"role" => "system",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => systemmsg),
|
||
]
|
||
),
|
||
Dict(
|
||
"role" => "user",
|
||
"content" => [
|
||
Dict("type" => "text", "text" => input),
|
||
]
|
||
),
|
||
],
|
||
"temperature" => 0.7
|
||
)
|
||
|
||
for attempt in 1:10
|
||
response = a.context.text2textInstructLLM(a.id, msg)
|
||
response = GeneralUtils.clean_json_response(response)
|
||
|
||
response = GeneralUtils.remove_french_accents(response)
|
||
think, response = GeneralUtils.extractthink(response)
|
||
responsedict = nothing
|
||
try
|
||
_responsedict = JSON.parse(response)
|
||
responsedict = GeneralUtils.dictify(_responsedict; keytype=String, sort_order=requiredKeys)
|
||
catch
|
||
println("\nERROR YiemAgent extractWineAttributes_2() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||
continue
|
||
end
|
||
|
||
# check whether all answer's key points are in responsedict
|
||
ispass, errormsg = GeneralUtils.checkAgentResponse_JSON(responsedict, requiredKeys)
|
||
if !ispass
|
||
errornote = errormsg
|
||
println("\nERROR YiemAgent extractWineAttributes_2() $errornote --(not qualify response)> $responsedict", @__FILE__, ":", @__LINE__, " $(Dates.now())\n")
|
||
continue
|
||
end
|
||
|
||
# delete some key words from responsedict
|
||
for (k, v) in responsedict
|
||
if k ∈ ["sweetness_keyword", "acidity_keyword", "tannin_keyword", "intensity_keyword"]
|
||
delete!(responsedict, k)
|
||
end
|
||
end
|
||
|
||
# get result in String. Reject "N/A" value
|
||
result = ""
|
||
for (k, v) in responsedict
|
||
if typeof(v) <: Number
|
||
result *= "$k: $v, "
|
||
elseif typeof(v) == String && !occursin("N/A", v)
|
||
result *= "$k: $v, "
|
||
end
|
||
end
|
||
result = result[1:end-2] # remove the ending ", "
|
||
|
||
@info "YiemAgent extractWineAttributes_2() " @__LINE__
|
||
@show result
|
||
@info "---\n" @__LINE__
|
||
|
||
return result
|
||
end
|
||
error("extractWineAttributes_2() failed to get a response")
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
|
||
"""
|
||
Get a simplified DDL schema for a PostgreSQL table with sample values.
|
||
|
||
Establishes a new connection and delegates to the connection-based overload.
|
||
|
||
# Arguments
|
||
- `pg_conn_str::String`: PostgreSQL connection string
|
||
- `table_name::String`: Name of the table to get schema for
|
||
|
||
# Keyword Arguments
|
||
- `schema_name::String="public"`: PostgreSQL schema name
|
||
|
||
# Returns
|
||
- `String`: Formatted DDL schema string with sample values
|
||
|
||
# Notes
|
||
- Delegates to `get_db_table_schema_simple_with_samples(conn, table_name, ...)` after creating
|
||
a new `LibPQ.Connection`.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> schema = YiemAgent.get_db_table_schema_simple_with_samples(conn_str, "wine")
|
||
"CREATE TABLE public.wine (\n wine_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),\n ...\n);"
|
||
```
|
||
"""
|
||
function get_db_table_schema_simple_with_samples(pg_conn_str::String, table_name::String;
|
||
schema_name::String="public")::String
|
||
conn = LibPQ.Connection(pg_conn_str)
|
||
return get_db_table_schema_simple_with_samples(conn, table_name; schema_name=schema_name)
|
||
end
|
||
|
||
"""
|
||
Get a simplified DDL schema for a PostgreSQL table with sample values.
|
||
|
||
Queries the PostgreSQL catalog for column metadata, fetches sample values, and builds
|
||
a DDL string with inline comments for each column.
|
||
|
||
# Arguments
|
||
- `conn`: An active PostgreSQL connection (e.g., `LibPQ.Connection`)
|
||
- `table_name::String`: Name of the table to get schema for
|
||
|
||
# Keyword Arguments
|
||
- `schema_name::String="public"`: PostgreSQL schema name
|
||
- `sample_count::Int=3`: Number of non-null sample values to fetch per column
|
||
|
||
# Returns
|
||
- `String`: Formatted DDL schema string in the style of `CREATE TABLE` statements
|
||
with sample values as inline comments
|
||
|
||
# Notes
|
||
- Queries `pg_attribute`, `pg_class`, `pg_namespace`, `pg_attrdef`, and `pg_constraint`
|
||
for column metadata, defaults, and constraints.
|
||
- Fetches up to `sample_count` non-null values per column via `json_agg`.
|
||
- Throws an error if the table is not found.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> schema = YiemAgent.get_db_table_schema_simple_with_samples(conn, "wine")
|
||
"CREATE TABLE public.wine (\n wine_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),\n ...\n);"
|
||
```
|
||
"""
|
||
function get_db_table_schema_simple_with_samples(conn, table_name::String; schema_name::String="public", sample_count::Int=3)::String
|
||
# 1. SQL query for catalog metadata
|
||
meta_sql = """
|
||
SELECT
|
||
a.attname AS column_name,
|
||
format_type(a.atttypid, a.atttypmod) AS data_type,
|
||
pg_get_expr(def.adbin, def.adrelid) AS default_value,
|
||
COALESCE(
|
||
(SELECT pg_get_constraintdef(p.oid)
|
||
FROM pg_catalog.pg_constraint p
|
||
WHERE p.conrelid = c.oid AND a.attnum = ANY(p.conkey)
|
||
LIMIT 1), ''
|
||
) AS constraint_definition
|
||
FROM pg_catalog.pg_attribute a
|
||
JOIN pg_catalog.pg_class c ON a.attrelid = c.oid
|
||
JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
|
||
LEFT JOIN pg_catalog.pg_attrdef def ON def.adrelid = c.oid AND def.adnum = a.attnum
|
||
WHERE c.relname = \$1
|
||
AND n.nspname = \$2
|
||
AND a.attnum > 0
|
||
AND NOT a.attisdropped
|
||
ORDER BY a.attnum;
|
||
"""
|
||
|
||
meta_res = DataFrame(execute(conn, meta_sql, [table_name, schema_name]))
|
||
|
||
if nrow(meta_res) == 0
|
||
error("Table '$schema_name.$table_name' not found.")
|
||
end
|
||
|
||
# 2. Build single dynamic query to fetch non-null samples for all columns
|
||
sample_selects = String[]
|
||
for row in eachrow(meta_res)
|
||
c_name = row.column_name
|
||
push!(sample_selects, """
|
||
(SELECT json_agg(s."$c_name")
|
||
FROM (
|
||
SELECT "$c_name"
|
||
FROM "$schema_name"."$table_name"
|
||
WHERE "$c_name" IS NOT NULL
|
||
LIMIT $sample_count
|
||
) s
|
||
) AS "$c_name"
|
||
""")
|
||
end
|
||
|
||
sample_sql = "SELECT " * join(sample_selects, ",\n ") * ";"
|
||
sample_df = DataFrame(execute(conn, sample_sql))
|
||
|
||
# 3. Build DDL definitions with inline sample comments
|
||
ddl_lines = String[]
|
||
constraints = String[]
|
||
|
||
for row in eachrow(meta_res)
|
||
col_name = row.column_name
|
||
data_type = row.data_type
|
||
default_val = ismissing(row.default_value) ? "" : " DEFAULT " * row.default_value
|
||
|
||
col_def = " \"$col_name\" $data_type$default_val"
|
||
|
||
# Fetch sample data for this column from the single-row sample DataFrame
|
||
samples_comment = ""
|
||
if nrow(sample_df) > 0
|
||
raw_samples = sample_df[1, Symbol(col_name)]
|
||
samples_str = ismissing(raw_samples) || isnothing(raw_samples) ? "[]" : string(raw_samples)
|
||
samples_comment = " -- Samples: $samples_str"
|
||
end
|
||
|
||
push!(ddl_lines, col_def * samples_comment)
|
||
|
||
# Handle table-level constraints
|
||
con_def = ismissing(row.constraint_definition) ? "" : row.constraint_definition
|
||
if !isempty(con_def) && !(con_def in constraints)
|
||
push!(constraints, " " * con_def)
|
||
end
|
||
end
|
||
|
||
all_definitions = vcat(ddl_lines, constraints)
|
||
body = join(all_definitions, ",\n")
|
||
|
||
return "CREATE TABLE \"$schema_name\".\"$table_name\" (\n$body\n);"
|
||
end
|
||
|
||
|
||
|
||
function classify_column(pg_conn_str::String, table_name::String, column_name::String;
|
||
sample_size::Integer=1000)
|
||
conn = LibPQ.Connection(pg_conn_str)
|
||
return classify_column(conn, table_name, column_name; sample_size=sample_size)
|
||
end
|
||
|
||
|
||
function classify_column(conn::LibPQ.Connection, table_name::String, column_name::String; sample_size::Int=1000)
|
||
# 1. Fetch BOTH data_type and udt_name (User Defined Type name)
|
||
meta_query = """
|
||
SELECT data_type, udt_name
|
||
FROM information_schema.columns
|
||
WHERE table_name = lower('$(table_name)')
|
||
AND column_name = lower('$(column_name)');
|
||
"""
|
||
|
||
pg_type = "unknown"
|
||
udt_name = "unknown"
|
||
|
||
try
|
||
df = DataFrame(LibPQ.execute(conn, meta_query))
|
||
if !isempty(df)
|
||
pg_type = df[1, :data_type]
|
||
udt_name = df[1, :udt_name]
|
||
end
|
||
catch e
|
||
@error "Failed to fetch metadata for $table_name.$column_name" exception=e
|
||
return "error"
|
||
end
|
||
|
||
# 2. FAST-TRACK: Check for pgvector FIRST
|
||
# pgvector registers as "USER-DEFINED" in data_type, but "vector" in udt_name
|
||
if udt_name == "vector"
|
||
return "semantic_search"
|
||
end
|
||
|
||
# 3. FAST-TRACK: Hard rules for standard non-text Postgres types
|
||
if pg_type in ["integer", "bigint", "smallint", "numeric", "real",
|
||
"double precision", "boolean", "date",
|
||
"timestamp without time zone", "timestamp with time zone", "uuid"]
|
||
return "exact_or_range"
|
||
end
|
||
|
||
# 4. SAMPLE: Get text statistics for remaining text columns
|
||
stats_query = """
|
||
SELECT
|
||
COUNT(*)::int AS total_count,
|
||
COUNT(DISTINCT $(column_name)::text)::int AS unique_count,
|
||
COALESCE(AVG(LENGTH($(column_name)::text)), 0)::float AS avg_len,
|
||
COALESCE(STDDEV(LENGTH($(column_name)::text)), 0)::float AS std_len
|
||
FROM (
|
||
SELECT $(column_name)
|
||
FROM $(table_name)
|
||
WHERE $(column_name) IS NOT NULL
|
||
LIMIT $sample_size
|
||
) AS sampled_data;
|
||
"""
|
||
|
||
try
|
||
df = DataFrame(LibPQ.execute(conn, stats_query))
|
||
if isempty(df) || df[1, :total_count] == 0
|
||
return "unknown"
|
||
end
|
||
|
||
total = df[1, :total_count]
|
||
unique = df[1, :unique_count]
|
||
avg_len = df[1, :avg_len]
|
||
std_len = df[1, :std_len]
|
||
ratio = unique / total
|
||
|
||
# 5. HEURISTICS: Route the column_name to the correct text bucket
|
||
return classify_text_column(unique, ratio, avg_len, std_len)
|
||
|
||
catch e
|
||
@warn "Failed to sample column_name $table_name.$column_name" exception=e
|
||
return "unknown"
|
||
end
|
||
end
|
||
|
||
# The Decision Tree for Text Columns (Unchanged, but kept for completeness)
|
||
function classify_text_column(unique_count::Integer, ratio::Float64, avg_len::Float64, std_len::Float64)
|
||
if avg_len > 60 && std_len > 25
|
||
return "full_text_search"
|
||
end
|
||
if ratio > 0.90 && avg_len < 40
|
||
return "exact_or_regex"
|
||
end
|
||
if unique_count <= 100
|
||
return "fuzzy_correction"
|
||
end
|
||
if ratio > 0.10 && avg_len < 35
|
||
return "fuzzy_correction"
|
||
end
|
||
if avg_len < 60
|
||
return "fuzzy_correction"
|
||
end
|
||
return "full_text_search"
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
end # module llmfunction |