diff --git a/etc.jl b/etc.jl index bc26067..1cc309e 100644 --- a/etc.jl +++ b/etc.jl @@ -109,3 +109,92 @@ end + + +function generate_wine_retail_sql(conditions::Dict{String, Any})::String + # 1. Base SQL structure + base_query = """ +SELECT + w.winery, + w.wine_name, + w.wine_id, + w.vintage, + w.region, + w.country, + w.wine_type, + w.grape, + w.serving_temperature, + w.sweetness, + w.intensity, + w.tannin, + w.acidity, + w.tasting_notes, + rw.price, + rw.currency, + w.image_url, + NULL AS retailer_name, + rw.retailer_id +FROM wine AS w +JOIN retailer_wine AS rw + ON w.wine_id = rw.wine_id +""" + + # 2. Dynamic WHERE Clause Builder + where_clauses = String[] + + # Iterate over each table condition provided + for (table_name, table_conditions) in conditions + + # Determine table alias + alias = if table_name == "wine" + "w" + elseif table_name == "retailer_wine" + "rw" + else + continue # Skip unsupported tables + end + + # Process condition dictionaries + if isa(table_conditions, Dict) && !isempty(table_conditions) + + for (column_name, filter_details) in table_conditions + + if isa(filter_details, Dict) && haskey(filter_details, "operator") + op = filter_details["operator"] + raw_val = filter_details["value"] + + # --- Value Type Handling --- + # Use tryparse instead of try/catch for cleaner, faster parsing + final_val = raw_val + + if op in ("=", "<", ">", "<=", ">=") + str_val = string(raw_val) + num_val = tryparse(Float64, str_val) + + if !isnothing(num_val) + final_val = isinteger(num_val) ? round(Int, num_val) : num_val + end + end + + # --- SQL Formatting --- + if isa(final_val, Number) + clause = "$(alias).$(column_name) $(op) $(final_val)" + else + # Escape single quotes within string values + escaped_val = replace(string(final_val), "'" => "''") + clause = "$(alias).$(column_name) $(op) '$(escaped_val)'" + end + + push!(where_clauses, clause) + end + end + end + end + + # 3. Assemble Final Query + where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ") + + return string(base_query, where_sql, ";") +end + + diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 270cac4..5c6c199 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -288,18 +288,7 @@ julia> thoughtdict = function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false )::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent} - # WORKING - # look_for_wine_in_wine_database(a, thoughtdict["action_input"]) - println("\ncheckinventory order: $(thoughtdict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - wineattributes_1 = extractWineAttributes_1(a, thoughtdict["action_input"]) - wineattributes_2 = extractWineAttributes_2(a, thoughtdict["action_input"]) - - retrieve_attributes = ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] - _inventoryquery = "$(thoughtdict["action_input"]), $wineattributes_1, $wineattributes_2, retailer_name: $(a.retailername), retailerid: $(a.retailerid)" - inventoryquery = "Retrieves $retrieve_attributes of wines that match the following criteria - {$_inventoryquery}" - println("\ncheckinventory input: $inventoryquery ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - if useSQLLLM # add suppport for similarSQLVectorDB textresult, result_raw = SQLLLM.query( @@ -313,7 +302,8 @@ function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool= else # direct query with possible sql instead of SQLLLM. - sql = generatesql(a, inventoryquery) + sql = predefined_wine_search_sql(a, thoughtdict["action_input"]) + # sql = generatesql(a, inventoryquery) println("\nSQL: $sql ", @__FILE__, ":", @__LINE__, " $(Dates.now()) \n") textresult, sql_result_df, success, _ = SQLexecution(a.context.executeSQL, sql) @@ -627,9 +617,9 @@ julia> thoughtdict = "action_name" => "SEARCH_WINE_DATABASE", "action_input" => "Brunello di Montalcino from Tenuta CastelGiocondo") ``` -julia> look_for_wine_in_wine_database(agent, thoughtdict["action_input"]) +julia> predefined_wine_search_sql(agent, thoughtdict["action_input"]) """ -function look_for_wine_in_wine_database(a::T, searchterm::String, +function predefined_wine_search_sql(a::T, searchterm::String, ; maxattempt=10 )::String where {T<:agent} @@ -731,6 +721,10 @@ function look_for_wine_in_wine_database(a::T, searchterm::String, response = a.context.text2textInstructLLM("random_id", msg) responsedict = Serde.parse_yaml(response) + # println("\n ", table_schema) + println("\n ", responsedict) + @info "before BM25 " @__LINE__ + """ responsedict = Dict( "wine" => Dict( @@ -758,83 +752,26 @@ function look_for_wine_in_wine_database(a::T, searchterm::String, for (column_name, v) in table_info_dict # - do_not_resolve_BM25_list = ["tasting_notes", "seo_name", "vintage", "grape"] - if column_name ∉ do_not_resolve_list + do_not_resolve_BM25_column = ["tasting_notes", "seo_name", "vintage", "grape", "price"] + if column_name ∉ do_not_resolve_BM25_column words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, table_name, column_name) resolved_word = GeneralUtils.resolve_entity(v["value"], words_catalog; threshold=0.9) - table_info_dict[column_name] = resolved_word + table_info_dict[column_name]["value"] = resolved_word + else + delete!(responsedict[table_name], column_name) + if length(responsedict[table_name]) == 0 + delete!(responsedict, table_name) + end end end end - - # check each attributes against each column in a database table with BM25 and get the closest - # word match because there is a typo sometimes. - for (k, v) in responsedict - if k ∉ ["tasting_notes"] - words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, "wine", k) - resolved_word = GeneralUtils.resolve_entity(v, words_catalog; threshold=0.9) - responsedict[k] = resolved_word - end - end - - - - - - - #WORKING println("\n", responsedict) - @info "test done " @__LINE__ - error(9999) + @info "after BM25 " @__LINE__ - - - - - think, response = GeneralUtils.extractthink(response) - responsedict = nothing - try - _responsedict = JSON.parse(response) - responsedict = GeneralUtils.dictify(_responsedict, keytype=String) - catch - println("\nERROR decisionMaker() failed to parse response: $response ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - continue - end - - # check each attributes against each column in a database table with BM25 and get the closest - # word match because there is a typo sometimes. - for (k, v) in responsedict - if k ∉ ["tasting_notes"] - words_catalog = GeneralUtils.harvest_entity_catalog(a.context.pg_conn_str, "wine", k) - resolved_word = GeneralUtils.resolve_entity(v, words_catalog; threshold=0.9) - responsedict[k] = resolved_word - end - end - - # LLM already extract user search term against tables schema - # Ex. responsedict = Dict( - # "wine_type"=> "red", # hard constraint - # "region"=> "bordeaux", # hard constraint - # "price_max"=> "100", # hard constraint - # "tasting_notes"=> "fruity, oak" # semantic search) + sql = predefined_wine_search_sql(responsedict) - - - - - - - - - - - - - - - - return items + return sql end error("SQLLLM DecisionMaker() failed to generate a thought \n", response) end @@ -889,6 +826,64 @@ function SQLexecution(executeSQL::Function, sql::T end end +function DEPRECIATED_search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false + )::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent} + + # XXX + predefined_wine_search_sql(a, thoughtdict["action_input"]) + + println("\ncheckinventory order: $(thoughtdict["action_input"]) ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + wineattributes_1 = extractWineAttributes_1(a, thoughtdict["action_input"]) + wineattributes_2 = extractWineAttributes_2(a, thoughtdict["action_input"]) + + retrieve_attributes = ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] + _inventoryquery = "$(thoughtdict["action_input"]), $wineattributes_1, $wineattributes_2, retailer_name: $(a.retailername), retailerid: $(a.retailerid)" + inventoryquery = "Retrieves $retrieve_attributes of wines that match the following criteria - {$_inventoryquery}" + println("\ncheckinventory input: $inventoryquery ", @__FILE__, ":", @__LINE__, " $(Dates.now())") + + if useSQLLLM + # add suppport for similarSQLVectorDB + textresult, result_raw = SQLLLM.query( + inventoryquery, + a.context.executeSQL, + a.context.text2textInstructLLM; + insertSQLVectorDB=a.context.insertSQLVectorDB, + similarSQLVectorDB=a.context.similarSQLVectorDB, + llmFormatName="qwen3") + thoughtdict["action_result"] = textresult + else + + # direct query with possible sql instead of SQLLLM. + sql = generatesql(a, inventoryquery) + println("\nSQL: $sql ", @__FILE__, ":", @__LINE__, " $(Dates.now()) \n") + textresult, sql_result_df, success, _ = SQLexecution(a.context.executeSQL, sql) + + items = nothing + if sql_result_df !== nothing + result_vec = GeneralUtils.dfToVectorDict(sql_result_df) + + # get image + for d in result_vec + image_url_json_str = d["image_url"] + image_url_json_obj = JSON.parse(image_url_json_str) + base_url = "http://192.168.88.106:8080/" + if haskey(image_url_json_obj, "bottle") + url = base_url * image_url_json_obj["bottle"] + image_data = HTTP.get(url) # vector{int} data + image_base64_string = base64encode(image_data.body) + d["image"] = image_base64_string + else + d["image"] = nothing + end + end + items = result_vec # image is added to each item + end + + thoughtdict["action_result"] = textresult + end + + return (thoughtdict=thoughtdict, result_raw=items) +end """ @@ -1475,7 +1470,92 @@ function jsoncorrection(config::T1, input::T2, correctJsonExample::T3; end +function predefined_wine_search_sql(conditions::Dict{String, Any})::String + # 1. Base SQL structure + base_query = +""" +SELECT + w.winery, + w.wine_name, + w.wine_id, + w.vintage, + w.region, + w.country, + w.wine_type, + w.grape, + w.serving_temperature, + w.sweetness, + w.intensity, + w.tannin, + w.acidity, + w.tasting_notes, + rw.price, + rw.currency, + w.image_url, + NULL AS retailer_name,66 + rw.retailer_id +FROM wine AS w +JOIN retailer_wine AS rw + ON w.wine_id = rw.wine_id +""" + # 2. Dynamic WHERE Clause Builder + where_clauses = String[] + + # Iterate over each table condition provided + for (table_name, table_conditions) in conditions + + # Determine table alias + alias = if table_name == "wine" + "w" + elseif table_name == "retailer_wine" + "rw" + else + continue # Skip unsupported tables + end + + # Process condition dictionaries + if isa(table_conditions, Dict) && !isempty(table_conditions) + + for (column_name, filter_details) in table_conditions + + if isa(filter_details, Dict) && haskey(filter_details, "operator") + op = filter_details["operator"] + raw_val = filter_details["value"] + + # --- Value Type Handling --- + # Use tryparse instead of try/catch for cleaner, faster parsing + final_val = raw_val + + if op in ("=", "<", ">", "<=", ">=") + str_val = string(raw_val) + num_val = tryparse(Float64, str_val) + + if !isnothing(num_val) + final_val = isinteger(num_val) ? round(Int, num_val) : num_val + end + end + + # --- SQL Formatting --- + if isa(final_val, Number) + clause = "$(alias).$(column_name) $(op) $(final_val)" + else + # Escape single quotes within string values + escaped_val = replace(string(final_val), "'" => "''") + clause = "$(alias).$(column_name) $(op) '$(escaped_val)'" + end + + push!(where_clauses, clause) + end + end + end + end + + # 3. Assemble Final Query + where_sql = isempty(where_clauses) ? "" : "WHERE " * join(where_clauses, " AND ") + + return string(base_query, where_sql, ";") +end diff --git a/src/type.jl b/src/type.jl index e42e8e3..a906fb6 100644 --- a/src/type.jl +++ b/src/type.jl @@ -23,75 +23,6 @@ end abstract type agent end -mutable struct companion <: agent - name::String # agent name - id::String # agent id - systemmsg::String # system message - tools::Dict # tools - maxHistoryMsg::Integer # e.g. 21th and earlier messages will get summarized - chathistory::Vector{Dict{String, Any}} - memory::Dict{String, Any} - context::NamedTuple # NamedTuple of functions - llmFormatName::String -end - -function companion( - context::agentcontext # NamedTuple of functions - ; - name::String= "Assistant", - id::String= GeneralUtils.uuid4snakecase(), - maxHistoryMsg::Integer= 20, - chathistory::Vector{Dict{String, String}} = Vector{Dict{String, String}}(), - llmFormatName::String= "granite3", - systemmsg::String= - """ - Your name: $name - Your sex: Female - Your role: You are a helpful assistant. - You should follow the following guidelines: - - Focus on the latest conversation. - - Your like to be short and concise. - - Let's begin! - """, - ) - - tools = Dict( # update input format - "CHAT_BOX"=> Dict( - "description" => "- CHAT_BOX which you can use to talk with the user. The input is your intentions for the dialogue. Be specific.", - ), - ) - - """ Memory - Ref: Chat prompt format https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/discussions/3 - NO "system" message in chathistory because I want to add it at the inference time - chathistory= [ - Dict("name"=>"user", "text"=> "Wassup!", "timestamp"=> Dates.now()), - Dict("name"=>"assistant", "text"=> "Hi I'm your assistant.", "timestamp"=> Dates.now()), - ] - """ - memory = Dict{String, Any}( - "events"=> Vector{Dict{String, Any}}(), - "state"=> Dict{String, Any}(), # state of the agent - "recap"=> OrderedDict{String, Any}(), # recap summary of the conversation - ) - - newAgent = companion( - name, - id, - systemmsg, - tools, - maxHistoryMsg, - chathistory, - memory, - context, - llmFormatName - ) - - return newAgent -end - - mutable struct sommelier <: agent name::String # agent name id::String # agent id @@ -210,11 +141,7 @@ function sommelier( memory = Dict{String, Any}( "shortmem"=> OrderedDict{String, Any}(), "scratchpad"=> "", - "events"=> Vector{Dict{String, Any}}(), - "state"=> Dict{String, Any}( - ), "recap"=> OrderedDict{String, Any}(), - ) newAgent = sommelier(