diff --git a/etc.jl b/etc.jl index 1cc309e..a35aae6 100644 --- a/etc.jl +++ b/etc.jl @@ -1,200 +1,10 @@ -using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, Dates, DataFrames, DataStructures -using GeneralUtils, SQLLLM, YiemAgent - -config = JSON.parsefile("./appconfig.json") -host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':') -port = parse(Int, _port) -dbname = "winedb" -user = config["externalservice"]["sommpanion_db"]["user"] -password = config["externalservice"]["sommpanion_db"]["password"] -pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password" - - function execute_sql_winedb(sql::T) where {T<:AbstractString} - host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':') - port = parse(Int, _port) - dbname = "winedb" - user = config["externalservice"]["sommpanion_db"]["user"] - password = config["externalservice"]["sommpanion_db"]["password"] - db_connection = LibPQ.Connection("host=$host_url port=$port dbname=$dbname user=$user password=$password") - result = nothing - try - result = LibPQ.execute(db_connection, sql) - catch e - LibPQ.close(db_connection) - end - - LibPQ.close(db_connection) - return result - end - - - -sql = -""" -SELECT T1.winery, T1.wine_name, T1.wine_id, T1.vintage, T1.region, T1.country, T1.wine_type, T1.grape, T1.serving_temperature, T1.sweetness, T1.intensity, T1.tannin, T1.acidity, T1.tasting_notes, T2.price, T2.currency, T1.image_url, T3.retailer_name, T3.retailer_id FROM "wine" AS T1 JOIN "retailer_wine" AS T2 ON T1.wine_id = T2.wine_id JOIN "retailer" AS T3 ON T2.retailer_id = T3.retailer_id WHERE T1.wine_name = 'Montrachet Grand Cru' AND T1.winery = 'Domaine Jacques Prieur' AND T3.retailer_name = 'Yiem Wines Ltd' AND T3.retailer_id = 'f54eab6b-7650-4448-b009-c53f3efbcc3b'; -""" - -textresult, sql_result_raw, _, _ = YiemAgent.SQLexecution(execute_sql_winedb, sql) -result_vec = GeneralUtils.dfToVectorDict(sql_result_raw) - -for d in result_vec - wine_name = d["wine_name"] - image_url_json_str = d["image_url"] - image_url_json_obj = JSON.parse(image_url_json) - base_url = "http://192.168.88.106:8080/" - image_base64 = - if haskey(image_url_json_obj, "bottle") - url = base_url * image_url_json_obj["bottle"] - image_data = HTTP.get(url) # vector{int} data - image_base64_string = base64encode(image_data) - else - nothing - end - d["image"] = image_base64 -end - - - - - - - - - - -using LibPQ -using Tables - -""" - update_car_regions_one_by_one(conn::LibPQ.Connection, target_word::String) - -Iterates through all rows in the 'car' table where the region is "German", -and updates them one-by-one to the `target_word`. -""" -function update_car_regions_one_by_one(pg_conn_str::String, replace_word::String , target_word::String) - conn = LibPQ.Connection(pg_conn_str) - # 1. Fetch the target rows. Assumes 'id' is the primary key. - # We select the ID to target rows individually during the update step. - select_query = "SELECT id FROM car WHERE region = '$replace_word';" - - result = execute(conn, select_query) - rows = Tables.rows(result) - - # 2. Prepare the update statement for execution reuse - # Using explicit types for parameter placeholders ($1, $2) - update_query = "UPDATE car SET region = \$1 WHERE id = \$2;" - - println("Starting one-by-one update...") - updated_count = 0 - - # 3. Iterate through rows one-by-one - for row in rows - # LibPQ row values are accessed via properties or column names - row_id = row.id - - # Execute the parameterized statement safely - execute(conn, update_query, [target_word, row_id]) - updated_count += 1 - end - - println("Successfully updated \$updated_count rows.") - return updated_count -end - - - - - - - - - - - -function generate_wine_retail_sql(conditions::Dict{String, Any})::String - # 1. Base SQL structure - base_query = """ -SELECT - w.winery, - w.wine_name, - w.wine_id, - w.vintage, - w.region, - w.country, - w.wine_type, - w.grape, - w.serving_temperature, - w.sweetness, - w.intensity, - w.tannin, - w.acidity, - w.tasting_notes, - rw.price, - rw.currency, - w.image_url, - NULL AS retailer_name, - rw.retailer_id -FROM wine AS w -JOIN retailer_wine AS rw - ON w.wine_id = rw.wine_id -""" - - # 2. Dynamic WHERE Clause Builder - where_clauses = String[] - - # Iterate over each table condition provided - for (table_name, table_conditions) in conditions - - # Determine table alias - alias = if table_name == "wine" - "w" - elseif table_name == "retailer_wine" - "rw" - else - continue # Skip unsupported tables - end - - # Process condition dictionaries - if isa(table_conditions, Dict) && !isempty(table_conditions) - - for (column_name, filter_details) in table_conditions - - if isa(filter_details, Dict) && haskey(filter_details, "operator") - op = filter_details["operator"] - raw_val = filter_details["value"] - - # --- Value Type Handling --- - # Use tryparse instead of try/catch for cleaner, faster parsing - final_val = raw_val - - if op in ("=", "<", ">", "<=", ">=") - str_val = string(raw_val) - num_val = tryparse(Float64, str_val) - - if !isnothing(num_val) - final_val = isinteger(num_val) ? round(Int, num_val) : num_val - end - end - - # --- SQL Formatting --- - if isa(final_val, Number) - clause = "$(alias).$(column_name) $(op) $(final_val)" - else - # Escape single quotes within string values - escaped_val = replace(string(final_val), "'" => "''") - clause = "$(alias).$(column_name) $(op) '$(escaped_val)'" - end - - push!(where_clauses, clause) - end - end - end - end - - # 3. Assemble Final Query - where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ") - - return string(base_query, where_sql, ";") -end - - +# check if this column has vector embedding. if there is one, seach vector version instead + column_name_embedding = column_name * "_embedding" + if occursin(column_name_embedding, tables_schema[column_name_embedding]) + vector_column = Dict( + "table_name"=> table_name, + "column_name"=> column_name_embedding, + "operator"=> "vector_similarity", + "value"=> column_obj["value"] + ) + end \ No newline at end of file diff --git a/src/interface.jl b/src/interface.jl index e1a333a..4750fca 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -163,7 +163,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=3 """ # strict output format - json_schema = Dict( + response_format = Dict( "type"=> "json_schema", "json_schema"=> Dict( "name"=> "user_profile", @@ -185,7 +185,7 @@ function decisionMaker(a::T; recentevents::Integer=20, maxattempt=3 "model"=> "gemma-4-E4B-it-UD-Q4_K_XL", "messages"=> a.chathistory, "temperature"=> 0.7, - "response_format"=> json_schema, + "response_format"=> response_format, ) for attempt in 1:maxattempt diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 4f0cfc9..2d7d41a 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -637,45 +637,47 @@ function predefined_wine_search_sql(a::T, searchterm::String, # your responsibility includes Fulfill the objective. - # you should only respond in JSON format as described below - { - table_name_1: - column_name_1: - operator: "=" - value: "..." - column_name_2: - operator: "=" - value: "..." - ... - table_name_2: - column_name_1: - operator: "=" - value: "..." - column_name_2: - operator: "=" - value: "..." - ... - } + # You must output your response as a JSON object containing a single key: "extracted_info". + The "extracted_info" key must contain an array of objects. Each object must contain: + 1) "table_name": The name of the table. + 2) "column_name": The specific column being filtered. + 3) "operator": The comparison operator (e.g., "=", ">", "LIKE"). + 4) "value": The value to compare against. + + If the user does not specify any filters, return an empty array for "extracted_info": {"extracted_info": []}. # here are some example 4-wheel drive car with red color that will give me fast and furious emotion. No more than 7000 USD - car_info: # table_name - drive_type: # column_name - operator: "=" # operator is not "N/A" because drive_type column store quantitative value - value: "4-wheel" # column_value - color: - operator: "=" # operator is not "N/A" because color column store quantitative value - value: "red" - drive_feeling: - operator: "N/A" # operator is "N/A" because drive_feeling column store qualitative value - value: "fast and furious" - price_list: - price: - operator: "<" # operator is not "N/A" because drive_type column store quantitative value - value: "7000" + { + "extracted_info": [ + { + "table_name": "car_info", + "column_name": "drive_type", + "operator": "=", + "value": "4-wheel" + }, + { + "table_name": "car_info", + "column_name": "color", + "operator": "=", + "value": "red" + }, + { + "table_name": "car_info", + "column_name": "drive_feeling", + "operator": "ILIKE", + "value": "fast and furious" + }, + { + "table_name": "price_list", + "column_name": "price", + "operator": "<", + "value": "7000" + } + } """ @@ -702,23 +704,47 @@ function predefined_wine_search_sql(a::T, searchterm::String, """ input = context * searchterm - json_schema = Dict( - "type"=> "json_schema", - "json_schema"=> Dict( - "name"=> "user_profile", - "strict"=> true, - "schema"=> Dict( - "type"=> "object", - "properties"=> Dict( - "plan"=> Dict("type"=> "string"), - "action_name"=> Dict("type"=> "string"), - "action_input"=> Dict("type"=> "string"), - ), - "required"=> ["plan", "action_name", "action_input"], - "additionalProperties"=> false + response_format = Dict( + "type" => "json_schema", + "json_schema" => Dict( + "name" => "extracted_conditions", + "strict" => true, + "schema" => Dict( + "type" => "object", + "properties" => Dict( + "extracted_info" => Dict( + "type" => "array", + "items" => Dict( + "type" => "object", + "properties" => Dict( + "table_name" => Dict( + "type" => "string", + "description" => "The name of the database table." + ), + "column_name" => Dict( + "type" => "string", + "description" => "The name of the column to filter on." + ), + "operator" => Dict( + "type" => "string", + "enum" => ["=", "!=", ">", "<", ">=", "<=", "LIKE", "IN", "IS NULL", "IS NOT NULL"], + "description" => "The SQL comparison operator." + ), + "value" => Dict( + "type" => ["string", "null"], + "description" => "The value to compare against. Use null for IS NULL/IS NOT NULL." + ) + ), + "required" => ["table_name", "column_name", "operator", "value"], + "additionalProperties" => false + ) + ) + ), + "required" => ["extracted_info"], + "additionalProperties" => false ) - ) ) +) msg = Dict( "model" => "gemma-4-E4B-it-UD-Q4_K_XL", @@ -736,85 +762,63 @@ function predefined_wine_search_sql(a::T, searchterm::String, ] ), ], - "temperature" => 0.7 + "temperature" => 0.7, + "response_format"=> response_format, ) for attempt in 1:maxattempt response = a.context.text2textInstructLLM("random_id", msg) - - responsedict = nothing - try - responsedict = Serde.parse_yaml(response) - catch e - println("\nERROR YiemAgent predefined_wine_search_sql() Error: $e --(not qualify response)-> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") - continue - end + responsedict = JSON.parse(response) + # responsedict = nothing + # try + # responsedict = Serde.parse_yaml(response) + # catch e + # println("\nERROR YiemAgent predefined_wine_search_sql() Error: $e --(not qualify response)-> $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") + # continue + # end # println("\n ", table_schema) println("\n ", responsedict) - @info "before BM25 " @__LINE__ + @info "before BM25 " @__LINE__ - """ - responsedict = Dict( - "wine" => Dict( - "tasting_notes" => Dict( - "operator" => "N/A", - "value" => "casual dinner" - ), - "wine_type" => Dict( - "operator" => "=", - "value" => "red" - ) - ), - "retailer_wine" => Dict( - "currency" => Dict( - "operator" => "=", "value" => "USD" - ), - "price" => Dict( - "operator" => "<", "value" => "1000" - ) - ) - ) - """ - - for (table_name, table_info_dict) in responsedict - for (column_name, v) in table_info_dict - bucket = classify_column(a.context.pg_conn_str, table_name, column_name) - - if bucket == "fuzzy_correction" - words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, table_name, column_name) - resolved_word = GeneralUtils.resolve_entity(v["value"], words_catalog; threshold=0.9) - table_info_dict[column_name]["value"] = resolved_word - end + #WORKING to ensure user input is correct + for entry in responsedict["extracted_info"] + table_name = entry["table_name"]::String + column_name = entry["column_name"]::String + + bucket = classify_column(a.context.pg_conn_str, table_name, column_name) + + if bucket == "fuzzy_correction" + words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, table_name, column_name) + resolved_word = GeneralUtils.resolve_entity(entry["value"], words_catalog; threshold=0.9) + entry["value"] = resolved_word end end - # filter for column that will be used for hard condition (SQL where clause) - # column with "N/A" operator will be used in vector search + # column with non-standard operator will be used in vector search vector_search_words = "" - for (table_name, table_dict) in responsedict - for (column_name, column_dict) in table_dict - if column_dict["operator"] ∉ ["=","<>","!=",">","<",">=","<=","!<","!>","<=>"] - vector_search_words = vector_search_words * column_dict["value"] * ", " - delete!(table_dict, column_name) + hard_operators = ["=","<>","!=",">","<",">=","<=","!<","!>","<=>"] - # remove table from responsedict if there is no column to used - if length(responsedict[table_name]) == 0 - delete!(responsedict, table_name) - end - end + # Build new list of hard condition entries + hard_conditions = JSON.Object{String, Any}[] + for entry in responsedict["extracted_info"] + if entry["operator"] ∈ hard_operators + push!(hard_conditions, entry) + else + vector_search_words = vector_search_words * entry["value"] * ", " end end + responsedict = hard_conditions println("") - pprintln(responsedict) + @show responsedict @info "predefined_wine_search_sql() " @__LINE__ - #WORKING do vector searched + #WORKING do vector search println("") @show vector_search_words - + # error(9999) sql = predefined_wine_search_sql(responsedict) return sql @@ -822,6 +826,83 @@ function predefined_wine_search_sql(a::T, searchterm::String, error("SQLLLM DecisionMaker() failed to generate a thought \n", response) end +function predefined_wine_search_sql(conditions::Vector{JSON.Object{String, Any}})::String + # 1. Base SQL structure + base_query = +""" +SELECT + w.winery, + w.wine_name, + w.wine_id, + w.vintage, + w.region, + w.country, + w.wine_type, + w.grape, + w.serving_temperature, + w.sweetness, + w.intensity, + w.tannin, + w.acidity, + w.tasting_notes, + rw.price, + rw.currency, + w.image_url, + r.retailer_name, + rw.retailer_id +FROM wine AS w +JOIN retailer_wine AS rw ON w.wine_id = rw.wine_id +JOIN retailer AS r ON rw.retailer_id = r.retailer_id +""" + + # 2. Dynamic WHERE Clause Builder + where_clauses = String[] + + # Iterate over each condition object in the array + for cond in conditions + table_name = String(cond["table_name"]) + column_name = String(cond["column_name"]) + op = String(cond["operator"]) + raw_val = cond["value"] + + # Determine table alias + alias = if table_name == "wine" + "w" + elseif table_name == "retailer_wine" + "rw" + else + continue + end + + # --- Value Type Handling --- + final_val = raw_val + + if op in ("=", "<", ">", "<=", ">=") + str_val = string(raw_val) + num_val = tryparse(Float64, str_val) + + if !isnothing(num_val) + final_val = isinteger(num_val) ? round(Int, num_val) : num_val + end + end + + # --- SQL Formatting --- + if isa(final_val, Number) + clause = "$(alias).$(column_name) $(op) $(final_val)" + else + escaped_val = replace(string(final_val), "'" => "''") + clause = "$(alias).$(column_name) $(op) '$(escaped_val)'" + end + + push!(where_clauses, clause) + end + + # 3. Assemble Final Query + where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ") + + return string(base_query, where_sql, ";") +end + function SQLexecution(executeSQL::Function, sql::T )::NamedTuple where {T<:AbstractString} @@ -1306,92 +1387,7 @@ function extractWineAttributes_2(a::T1, input::T2)::String where {T1<:agent, T2< error("extractWineAttributes_2() failed to get a response") end -function predefined_wine_search_sql(conditions::Dict{String, Any})::String - # 1. Base SQL structure - base_query = -""" -SELECT - w.winery, - w.wine_name, - w.wine_id, - w.vintage, - w.region, - w.country, - w.wine_type, - w.grape, - w.serving_temperature, - w.sweetness, - w.intensity, - w.tannin, - w.acidity, - w.tasting_notes, - rw.price, - rw.currency, - w.image_url, - r.retailer_name, - rw.retailer_id -FROM wine AS w -JOIN retailer_wine AS rw ON w.wine_id = rw.wine_id -JOIN retailer AS r ON rw.retailer_id = r.retailer_id -""" - # 2. Dynamic WHERE Clause Builder - where_clauses = String[] - - # Iterate over each table condition provided - for (table_name, table_conditions) in conditions - - # Determine table alias - alias = if table_name == "wine" - "w" - elseif table_name == "retailer_wine" - "rw" - else - continue # Skip unsupported tables - end - - # Process condition dictionaries - if isa(table_conditions, Dict) && !isempty(table_conditions) - - for (column_name, filter_details) in table_conditions - - if isa(filter_details, Dict) && haskey(filter_details, "operator") - op = filter_details["operator"] - raw_val = filter_details["value"] - - # --- Value Type Handling --- - # Use tryparse instead of try/catch for cleaner, faster parsing - final_val = raw_val - - if op in ("=", "<", ">", "<=", ">=") - str_val = string(raw_val) - num_val = tryparse(Float64, str_val) - - if !isnothing(num_val) - final_val = isinteger(num_val) ? round(Int, num_val) : num_val - end - end - - # --- SQL Formatting --- - if isa(final_val, Number) - clause = "$(alias).$(column_name) $(op) $(final_val)" - else - # Escape single quotes within string values - escaped_val = replace(string(final_val), "'" => "''") - clause = "$(alias).$(column_name) $(op) '$(escaped_val)'" - end - - push!(where_clauses, clause) - end - end - end - end - - # 3. Assemble Final Query - where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ") - - return string(base_query, where_sql, ";") -end diff --git a/src/type.jl b/src/type.jl index fccda67..f960b78 100644 --- a/src/type.jl +++ b/src/type.jl @@ -164,7 +164,6 @@ function sommelier( - You can only recommend wines that are currently in our inventory - Before searching the database for wine, ensure you have at least the following information: 1) budget, 2) wine type, and 3) occasion. Additional details are always helpful. If the user is unsure, provide relevant information and gather insights to make reasonable inferences. - Ask the user one question at a time. - - Do not ask the user about wine's flavor e.g. floral, citrusy, nutty or some thing similar as these terms cannot be used to search the database. - Once the user has selected their wine, if you haven't already, ask the user whether they need any further assistance. Do not offer any additional services. - Only end the conversation when the user explicitly intends to do so. When ending, ensure a polite farewell and an invitation to return in the future. - Spicy foods should be paired only with light red wines.