From edad4422427071a52024e76713cfb2615189564d Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 13 Jul 2026 21:06:09 +0700 Subject: [PATCH 01/10] update --- etc.jl | 355 ++++++++++++++++++++++++++++ src/llmUtil.jl | 614 ++++++++++++++++++++++--------------------------- 2 files changed, 631 insertions(+), 338 deletions(-) create mode 100644 etc.jl diff --git a/etc.jl b/etc.jl new file mode 100644 index 0000000..894b9a3 --- /dev/null +++ b/etc.jl @@ -0,0 +1,355 @@ + + +using LibPQ +using DataFrames + +""" + extract_vector_metadata(pg_conn_str::String) -> DataFrame + +Queries PostgreSQL system catalogs to extract a rich semantic text map of every +column in the database. Returns a DataFrame designed for vector embedding generation. +""" +function extract_vector_metadata(pg_conn_str::String) + conn = LibPQ.Connection(pg_conn_str) + + # This direct SQL query pulls the column specifications along with column-level descriptions + query = """ + SELECT + c.relname AS table_name, + a.attname AS column_name, + format_type(a.atttypid, a.atttypmod) AS data_type, + COALESCE(d.description, '') AS column_description, + CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, + CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + -- Join to fetch column comments/descriptions + LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum + -- Check if column is part of a Primary Key + LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid + AND pk.contype = 'p' + AND a.attnum = ANY(pk.conkey) + -- Check if column is part of a Foreign Key + LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid + AND fk.contype = 'f' + AND a.attnum = ANY(fk.conkey) + WHERE + n.nspname = 'public' -- Only user schemas + AND c.relkind = 'r' -- Only standard tables + AND a.attnum > 0 -- Skip system hidden columns + AND NOT a.attisdropped; -- Skip dropped columns + """ + + try + # Execute and format into a clean DataFrame + result = execute(conn, query) + df = DataFrame(result) + + # Create a unique document ID for each vector row + df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)] + + return df + finally + close(conn) + end +end + + + +""" + generate_embedding_payloads(df::DataFrame) -> Vector{Dict} + +Transforms the metadata DataFrame into structured text strings optimal for +vector space mapping. +""" +function generate_embedding_payloads(df::DataFrame) + payloads = Dict[] + + for row in eachrow(df) + # 1. Build a rich text description summarizing the column's role + text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)" + + if row.is_primary_key + text_payload *= " [PRIMARY KEY]" + end + if row.is_foreign_key + text_payload *= " [FOREIGN KEY RELATIONAL LINK]" + end + + # Append business descriptions if they exist in the database comments + if !isempty(strip(row.column_description)) + text_payload *= " | Description: $(row.column_description)" + else + text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture." + end + + # 2. Package everything neatly to be passed to your vector store client + push!(payloads, Dict( + "id" => row.vector_id, + "text_content" => text_payload, + "metadata" => Dict( + "table" => row.table_name, + "column" => row.column_name, + "type" => row.data_type + ) + )) + end + + return payloads +end + + +""" + resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String} + +Takes a scattered array of semantically matched tables from Stage 1, navigates +the undirected network structure, and isolates the minimum interconnected subgraph +required to weave ALL hits into a single valid SQL query. +""" +function resolve_semantic_cluster( + vector_hits::Vector{String}, + g::SimpleGraph, + table_to_id::Dict{String, Int}, + id_to_table::Dict{Int, String} + ) + # Filter out hits that don't exist in our actual database graph mapping + valid_node_ids = Int[] + for hit in vector_hits + if haskey(table_to_id, hit) + push!(valid_node_ids, table_to_id[hit]) + else + @warn "Vector hit '$hit' does not map to an existing database table." + end + end + + unique!(valid_node_ids) + + # Edge Case Handlers + if isempty(valid_node_ids) + return String[] + elseif length(valid_node_ids) == 1 + return [id_to_table[valid_node_ids[1]]] + end + + # The Isolated Subgraph Set to build our final context + schema_subgraph_nodes = Set{Int}() + + # Phase A: Select an initial anchor component. We use the highest-ranked vector hit. + anchor_node = valid_node_ids[1] + push!(schema_subgraph_nodes, anchor_node) + + # Phase B: Sequentially route paths to all other semantic coordinates + for target_node in valid_node_ids[2:end] + # Skip if an earlier loop trajectory already naturally absorbed this table + if target_node in schema_subgraph_nodes + continue + end + + # Calculate the shortest path tree from the CURRENT state of our subgraph + # We find the shortest path from the target back to ANY node currently in our tree + shortest_paths = dijkstra_shortest_paths(g, target_node) + + # Find which node currently in our subgraph is closest to the target node + closest_subgraph_node = 0 + min_distance = Inf + + for subgraph_node in schema_subgraph_nodes + dist = shortest_paths.dists[subgraph_node] + if dist < min_distance + min_distance = dist + closest_subgraph_node = subgraph_node + end + end + + # Reconstruct the path from the target node to the closest point on our existing tree + if closest_subgraph_node != 0 + curr = closest_subgraph_node + while curr != 0 + push!(schema_subgraph_nodes, curr) + curr = shortest_paths.parents[curr] + if curr == target_node + push!(schema_subgraph_nodes, target_node) + break + end + end + end + end + + # Map the unique structural nodes back to clean table names + return [id_to_table[node_id] for node_id in schema_subgraph_nodes] +end + +function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String}) + documents_dict = Dict("documents" => text) + payloads = [("documents", documents_dict, "dictionary")] + _, msg_envelope_json_str = msghandler.smartpack( + config["externalservice"]["servicesloadbalancer"]["nats"], + payloads; + msg_purpose="embedding", + broker_url=config["nats_server_info"]["url"], + fileserver_url=config["externalservice"]["fileserver"]["url"]) + + reply = NATS.request(nats_conn, + config["externalservice"]["servicesloadbalancer"]["nats"], + msg_envelope_json_str, timeout=120) + incoming_env_json_str = String(reply.payload) + incoming_env = msghandler.smartunpack(incoming_env_json_str) + embedding_response = incoming_env["payloads"][1][2] + + return embedding_response +end + +nats_conn = NATS.connect(config["nats_server_info"]["url"]) + +# Run the extractor +metadata_df = extract_vector_metadata(pg_conn_str) +embedding_ready = generate_embedding_payloads(metadata_df) + +println(embedding_ready[1]["text_content"]) +# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items." + +embedding_ready_2 = [i["text_content"] for i in embedding_ready] +table_embedding = get_embedding(nats_conn, embedding_ready_2) + +user_question = + """ + Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b} + """ +user_question_embedding = get_embedding(nats_conn, [user_question]) + + +using Distances +similarity = 1 - cosine_dist(Float64.(table_embedding["data"][1]["embedding"]), + Float64.(user_question_embedding["data"][1]["embedding"]) + ) +user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"]) +user_question_similarity = [] +for i in table_embedding["data"] + i_data = i["embedding"] + i_float = Float64.(i_data) + r = 1 - cosine_dist(i_float, user_question_embedding) + push!(user_question_similarity, r) +end + +new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) +sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min + +# top 20 of sorted_df get this tables +vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"] + +g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str) + +# tables that I should put schema in LLM context +optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) + + + +function related_tables_for_user_question() + + + + +end + +# ---------------------------------------------- 100 --------------------------------------------- # + + +# Agent 3 (The Entity Resolver): Instantly runs a fast, local token search (like BM25) to map messy user text (like HandOld) to the exact database string (Hand Old Bar & Grill) before the SQL is drafted. + +using StringDistances + +""" + harvest_entity_catalog(conn_str::String, table::String, column::String) -> Vector{String} + +Pulls unique, clean text strings from a specific entity column to build a local index. +""" +function harvest_entity_catalog(conn_str::String, table::String, column::String) + conn = LibPQ.Connection(conn_str) + + # We only care about unique, non-null values to keep the index fast and dense + query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" + + try + df = DataFrame(execute(conn, query)) + # Return as a clean array of strings + return String.(strip.(df[:, 1])) + finally + close(conn) + end +end + + +""" + resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.6) -> String + +Parses user text, matches it against the real database catalog, and returns +the exact string found in the database. Returns an empty string if no confident match. +""" +function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5) + best_match = "" + highest_score = 0.0 + + # Normalize input text to ensure case-insensitive matching + clean_input = lowercase(strip(messy_input)) + + for real_string in catalog + clean_real = lowercase(real_string) + + # Calculate phonetic/structural similarity score (0.0 to 1.0) + # JaroWinkler is optimized for short strings, names, and partial acronyms + score = compare(clean_real, clean_input, JaroWinkler()) + + # Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill" + # We strip spaces to check if the user just compressed words together + if contains(replace(clean_real, " " => ""), clean_input) + score = max(score, 0.85) + end + + if score > highest_score + highest_score = score + best_match = real_string + end + end + + # Only return if we cross our safety confidence barrier + if highest_score >= threshold + return best_match + end + + return "" # No confident match found +end + + + +winery_catalog = harvest_entity_catalog(conn_str, "wine", "winery") +# Let's assume the catalog contains: ["Hand Old Bar & Grill", "Bangkok Diner", "Phuket Seafood"] + +# 2. The user asks a messy question with a typo and compressed text +user_question = "What are the total sales at HandOld last week?" + +# 3. Agent 3 isolates potential nouns or scans the question against the index +# We look for words that don't match standard english dictionary tokens, or check the full string segments +detected_entity = "Jacob" + +# 4. Run the resolution engine +exact_db_string = resolve_entity(detected_entity, winery_catalog) +# "United States" + +println("Messy Input: ", detected_entity) +println("Resolved Engine Value: ", exact_db_string) +# Output: "Hand Old Bar & Grill" + + + + + + + + + + + + + + diff --git a/src/llmUtil.jl b/src/llmUtil.jl index 95dda24..be57fba 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -1,103 +1,12 @@ module llmUtil -export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection, deFormatLLMtext, extractthink, +export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response -using UUIDs, JSON, Dates +using UUIDs, JSON, Dates, DataFrames using GeneralUtils # ---------------------------------------------- 100 --------------------------------------------- # -#[PENDING] update code to use JSON - -""" Convert a single chat dictionary into LLM model instruct format. - -# Llama 3 instruct format example - <|begin_of_text|> - <|start_header_id|>system<|end_header_id|> - You are a helpful assistant. - <|eot_id|> - <|start_header_id|>user<|end_header_id|> - Get me an icecream. - <|eot_id|> - <|start_header_id|>assistant<|end_header_id|> - Go buy it yourself at 7-11. - <|eot_id|> - -# Arguments - - `name::T` - message owner name e.f. "system", "user" or "assistant" - - `text::T` - -# Return - - `formattedtext::String` - text formatted to model format - -# Example -```jldoctest -julia> using Revise -julia> using YiemAgent -julia> d = Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",) -julia> formattedtext = YiemAgent.formatLLMtext_llama3instruct(d[:name], d[:text]) -"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n" -``` - -Signature -""" -function formatLLMtext_llama3instruct(name::T, text::T; - assistantStarter::Bool=false) where {T<:AbstractString} - formattedtext = - if name == "system" - """ - <|start_header_id|>$name<|end_header_id|> - $text - <|eot_id|> - """ - else - """ - <|start_header_id|>$name<|end_header_id|> - $text - <|eot_id|> - """ - end - - if assistantStarter - formattedtext *= - """ - <|start_header_id|>assistant<|end_header_id|> - """ - end - - return formattedtext -end - - -function formatLLMtext_qwen(name::T, text::T; - assistantStarter::Bool=false) where {T<:AbstractString} - formattedtext = - if name == "system" - """ - <|im_start|>$name - $text - <|im_end|> - """ - else - """ - <|im_start|>$name - $text - <|im_end|> - """ - end - - if assistantStarter - formattedtext *= - """ - <|im_start|>assistant - """ - end - - return formattedtext -end - function formatLLMtext_qwen3(name::T, text::T; assistantStarter::Bool=false) where {T<:AbstractString} @@ -127,59 +36,6 @@ function formatLLMtext_qwen3(name::T, text::T; end -function formatLLMtext_phi4(name::T, text::T; - assistantStarter::Bool=false) where {T<:AbstractString} - formattedtext = - if name == "system" - """ - <|system|> - $text - <|end|> - """ - else - """ - <|assistant|> - $text - <|end|> - """ - end - - if assistantStarter - formattedtext *= - """ - <|assistant|> - """ - end - - return formattedtext -end - - -function formatLLMtext_granite3(name::T, text::T; - assistantStarter::Bool=false) where {T<:AbstractString} - formattedtext = - if name == "system" - """ - <|start_of_role|>system<|end_of_role|>{$text}<|end_of_text|> - """ - else - """ - <|start_of_role|>$name<|end_of_role|>{$text}<|end_of_text|> - """ - end - - if assistantStarter - formattedtext *= - """ - <|start_of_role|>assistant<|end_of_role|>{ - """ - end - - return formattedtext -end - - - """ Convert a vector of chat message dictionaries into LLM model instruct format. # Arguments @@ -194,13 +50,13 @@ end # Example ```jldoctest julia> using Revise -julia> using YiemAgent +julia> using GeneralUtils julia> chatmessage = [ Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",), Dict(:name=> "user",:text=> "list me all planets in our solar system.",), Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",), ] -julia> formattedtext = YiemAgent.formatLLMtext(chatmessage, "llama3instruct") +julia> formattedtext = GeneralUtils.formatLLMtext(chatmessage, "llama3instruct") "<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n <|start_header_id|>user<|end_header_id|>\n list me all planets in our solar system.\n <|eot_id|>\n <|start_header_id|>assistant<|end_header_id|>\n I'm sorry. I don't know. You tell me.\n <|eot_id|>\n" ``` """ @@ -237,192 +93,6 @@ function formatLLMtext(messages::Vector{Dict{Symbol, T}}, formatname::String return str end -""" Revert LLM-format response back into regular text. - -# Arguments - - `text::String` - The LLM formatted string to be converted. - -# Return - - `normalText::String` - The original plain text extracted from the given LLM-formatted string. - -# Example -```jldoctest -julia> using Revise -julia> using YiemAgent -julia> response = "<|begin_of_text|>This is a sample system instruction.<|eot_id|>" -julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3") -"This is a sample system instruction." -``` -""" -function deFormatLLMtext(text::String, formatname::String; includethink::Bool=false - )::String - f = - if formatname == "granite3" - deFormatLLMtext_granite3 - elseif formatname == "qwen3" - deFormatLLMtext_qwen3 - else - error("$formatname template not define yet") - end - - r = f(text) - result = r === nothing ? text : r - return result -end - - -""" Revert LLM-format response back into regular text for Granite 3 format. - -# Arguments - - `text::String` - The LLM formatted string to be converted. - -# Return - - `normalText::Union{Nothing, String}` - The original plain text extracted from the given LLM-formatted string. - Returns nothing if the text is not in Granite 3 format. - -# Example -```jldoctest -julia> using Revise -julia> using YiemAgent -julia> response = "{This is a sample LLM response.}" -julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3") -"This is a sample LLM response." -""" -function deFormatLLMtext_granite3(text::String)::Union{Nothing, String} - # check if '{' and '}' are in the text because it's a special format for the LLM response - if contains(text, "<|im_start|>assistant") - # get the text between '{' and '}' - text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1] - return text_between_braces - elseif text[end] == '}' - text = "{$text" - text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1] - else - return nothing - end -end - - -function deFormatLLMtext_qwen3(text::String)::Union{Nothing, String} - return text -end - -# function deFormatLLMtext_qwen3(text::String; includethink::Bool=false)::Union{Nothing, String} -# think = nothing -# str = nothing - -# if occursin("", text) -# r = GeneralUtils.extractTextBetweenString(text, "", "") -# if r[:success] -# think = r[:text] -# end -# str = string(split(text, "")[2]) -# end - -# if includethink == true && occursin("", text) -# result = "ModelThought: $think $str" -# return result -# elseif includethink == false && occursin("", text) -# result = str -# return result -# else -# return text -# end -# end - - -""" Attemp to correct LLM response's incorrect JSON response. - -# Arguments - - `a::T1` - one of Yiem's agent - - `input::T2` - text to be send to virtual wine customer - -# Return - - `correctjson::String` - corrected json string - -# Example -```jldoctest -julia> -``` - -# Signature -""" -function jsoncorrection(config::T1, input::T2, correctJsonExample::T3; - maxattempt::Integer=3 - ) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString} - - incorrectjson = deepcopy(input) - correctjson = nothing - - for attempt in 1:maxattempt - try - d = copy(JSON3.read(incorrectjson)) - correctjson = incorrectjson - return correctjson - catch e - @warn "Attempting to correct JSON string. Attempt $attempt" - e = """$e""" - if occursin("EOF", e) - e = split(e, "EOF")[1] * "EOF" - end - incorrectjson = deepcopy(input) - _prompt = - """ - Your goal are: - 1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function. - 2) Provide Corrected JSON string only. Do not provide any other info. - - $correctJsonExample - - Let's begin! - Given JSON string: $incorrectjson - The given JSON string failed to load previously because: $e - Corrected JSON string: - """ - - # apply LLM specific instruct format - externalService = config[:externalservice][:text2textinstruct] - llminfo = externalService[:llminfo] - prompt = - if llminfo[:name] == "llama3instruct" - formatLLMtext_llama3instruct("system", _prompt) - else - error("llm model name is not defied yet $(@__LINE__)") - end - - # send formatted input to user using GeneralUtils.sendReceiveMqttMsg - msgMeta = GeneralUtils.generate_msgMeta( - externalService[:mqtttopic], - senderName= "jsoncorrection", - senderId= uuid4snakecase(), - receiverName= "text2textinstruct", - mqttBroker= config[:mqttServerInfo][:broker], - mqttBrokerPort= config[:mqttServerInfo][:port], - ) - - outgoingMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict( - :text=> prompt, - :kwargs=> Dict( - :max_tokens=> 512, - :stop=> ["<|eot_id|>"], - ) - ) - ) - result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120) - incorrectjson = result[:response][:text] - end - end -end - function extractthink(text::String) think = nothing @@ -470,18 +140,18 @@ The validation logic checks: # Example ```julia -julia> using YiemAgent +julia> using GeneralUtils julia> requiredKeys = ["wine_name", "price", "rating"] julia> response = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98) -julia> YiemAgent.checkAgentResponse_JSON(response, requiredKeys) +julia> GeneralUtils.checkAgentResponse_JSON(response, requiredKeys) (true, nothing) julia> response_missing = Dict("wine_name"=>"Château Margaux", "price"=>250.0) -julia> YiemAgent.checkAgentResponse_JSON(response_missing, requiredKeys) +julia> GeneralUtils.checkAgentResponse_JSON(response_missing, requiredKeys) (false, "rating are missing from your previous response") julia> response_extra = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98, "extra_field"=>"data") -julia> YiemAgent.checkAgentResponse_JSON(response_extra, requiredKeys) +julia> GeneralUtils.checkAgentResponse_JSON(response_extra, requiredKeys) (false, "Your previous attempt has duplicated points according to the required response format") ``` """ @@ -550,12 +220,280 @@ function clean_json_response(text::String) end +""" Extract vector metadata from PostgreSQL database. + +Queries PostgreSQL system catalogs to extract column metadata including table names, +column names, data types, descriptions, and constraint information (primary/foreign keys). +Returns a DataFrame designed for vector embedding generation. + +# Arguments +- `pg_conn_str::String` + PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname") + +# Return +- `DataFrame` + A DataFrame with columns: + - `table_name`: Name of the table + - `column_name`: Name of the column + - `data_type`: PostgreSQL data type with typemod + - `column_description`: Column's comment/description (empty string if none) + - `is_primary_key`: Boolean indicating if column is part of primary key + - `is_foreign_key`: Boolean indicating if column is part of foreign key + - `vector_id`: Generated unique identifier in format `col_{table_name}_{column_name}` + +# Example +```julia +julia> using GeneralUtils +julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb" +julia> df = GeneralUtils.extract_vector_metadata(pg_conn) +DataFrame +6 rows × 7 columns +table_name column_name data_type column_description is_primary_key is_foreign_key vector_id +─────────────┬───────────┬───────────┬───────────────────┬───────────────┬───────────────┬──────────────────────── +users id integer User ID true false col_users_id +users name text User name false false col_users_name +users email text User email false false col_users_email +products id integer Product ID true false col_products_id +products price numeric Product price false false col_products_price +products user_id integer Reference to user false true col_products_user_id +``` +""" +function extract_vector_metadata(pg_conn_str::String)::DataFrame + conn = LibPQ.Connection(pg_conn_str) + + # This direct SQL query pulls the column specifications along with column-level descriptions + query = + """ + SELECT + c.relname AS column_name, + format_type(a.atttypid, a.atttypmod) AS data_type, + COALESCE(d.description, '') AS column_description, + CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, + CASE WHEN AS table_name, + a.attnamefk.contype = 'f' THEN true ELSE false END AS is_foreign_key + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + -- Join to fetch column comments/descriptions + LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum + -- Check if column is part of a Primary Key + LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid + AND pk.contype = 'p' + AND a.attnum = ANY(pk.conkey) + -- Check if column is part of a Foreign Key + LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid + AND fk.contype = 'f' + AND a.attnum = ANY(fk.conkey) + WHERE + n.nspname = 'public' -- Only user schemas + AND c.relkind = 'r' -- Only standard tables + AND a.attnum > 0 -- Skip system hidden columns + AND NOT a.attisdropped; -- Skip dropped columns + """ + + try + # Execute and format into a clean DataFrame + result = execute(conn, query) + df = DataFrame(result) + + # Create a unique document ID for each vector row + df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)] + + return df + finally + close(conn) + end +end +""" Generate embedding payloads from vector metadata DataFrame. + +Transforms the metadata DataFrame from `extract_vector_metadata` into a vector of +dictionaries structured for vector embedding storage and retrieval. + +# Arguments +- `df::DataFrame` + A DataFrame with columns from `extract_vector_metadata`: `table_name`, `column_name`, + `data_type`, `column_description`, `is_primary_key`, `is_foreign_key`, `vector_id`. + +# Return +- `Vector{Dict}` + A vector of dictionaries with keys: + - `id`: The vector ID from `vector_id` + - `text_content`: A structured text string combining table, column, type, constraint + markers, and description + - `metadata`: A dictionary containing `table`, `column`, and `type` + +# Details +The function constructs rich text payloads by: +1. Building a base string with table name, column name, and data type +2. Appending constraint markers `[PRIMARY KEY]` or `[FOREIGN KEY RELATIONAL LINK]` +3. Adding column description if available, otherwise generating a default description + +# Example +```julia +julia> using GeneralUtils +julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb" +julia> df = GeneralUtils.extract_vector_metadata(pg_conn) +julia> payloads = GeneralUtils.generate_embedding_payloads(df) +3-element Vector{Dict}: + Dict("id" => "col_users_id", "text_content" => "Table: users | Column: id | Type: integer [PRIMARY KEY] | Description: User ID", "metadata" => Dict("table" => "users", "column" => "id", "type" => "integer")) + Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text")) + Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text")) +``` +""" +function generate_embedding_payloads(df::DataFrame)::Vector{Dict} + payloads = Dict[] + + for row in eachrow(df) + # 1. Build a rich text description summarizing the column's role + text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)" + + if row.is_primary_key + text_payload *= " [PRIMARY KEY]" + end + if row.is_foreign_key + text_payload *= " [FOREIGN KEY RELATIONAL LINK]" + end + + # Append business descriptions if they exist in the database comments + if !isempty(strip(row.column_description)) + text_payload *= " | Description: $(row.column_description)" + else + text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture." + end + + # 2. Package everything neatly to be passed to your vector store client + push!(payloads, Dict( + "id" => row.vector_id, + "text_content" => text_payload, + "metadata" => Dict( + "table" => row.table_name, + "column" => row.column_name, + "type" => row.data_type + ) + )) + end + + return payloads +end +""" Resolve semantic cluster from vector hits using graph traversal. +Finds the minimum interconnected subgraph that connects all semantically matched tables +from vector search results, enabling construction of valid SQL queries across related tables. +# Arguments +- `vector_hits::Vector{String}` + A vector of table names matched semantically from Stage 1 vector search. +- `g::SimpleGraph` + An undirected graph representing table relationships (nodes=tables, edges=foreign key relations). +- `table_to_id::Dict{String, Int}` + Mapping from table names to node IDs in the graph. +- `id_to_table::Dict{Int, String}` + Reverse mapping from node IDs to table names. + +# Return +- `Vector{String}` + A vector of table names representing the minimum subgraph that connects all input vector hits. + The order reflects traversal path from anchor node to connected components. + +# Details +The algorithm: +1. Validates vector hits against the graph's table mapping +2. Handles edge cases: empty hits, single table (returns as-is) +3. Uses the highest-ranked vector hit as anchor node +4. For each remaining hit, finds shortest path to current subgraph using Dijkstra's algorithm +5. Builds minimal connected subgraph containing all hits +6. Returns table names in traversal order + +# Example +```julia +julia> using GeneralUtils, Graphs +julia> g = SimpleGraph(5) +julia> add_edge!(g, 1, 2) +julia> add_edge!(g, 2, 3) +julia> add_edge!(g, 3, 4) +julia> table_to_id = Dict("users" => 1, "orders" => 2, "payments" => 3, "products" => 4, "inventory" => 5) +julia> id_to_table = Dict(1 => "users", 2 => "orders", 3 => "payments", 4 => "products", 5 => "inventory") +julia> vector_hits = ["users", "products"] +julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) +["users", "orders", "products"] +``` +""" +function resolve_semantic_cluster( + vector_hits::Vector{String}, + g::SimpleGraph, + table_to_id::Dict{String, Int}, + id_to_table::Dict{Int, String} + )::Vector{String} + # Filter out hits that don't exist in our actual database graph mapping + valid_node_ids = Int[] + for hit in vector_hits + if haskey(table_to_id, hit) + push!(valid_node_ids, table_to_id[hit]) + else + @warn "Vector hit '$hit' does not map to an existing database table." + end + end + + unique!(valid_node_ids) + + # Edge Case Handlers + if isempty(valid_node_ids) + return String[] + elseif length(valid_node_ids) == 1 + return [id_to_table[valid_node_ids[1]]] + end + + # The Isolated Subgraph Set to build our final context + schema_subgraph_nodes = Set{Int}() + + # Phase A: Select an initial anchor component. We use the highest-ranked vector hit. + anchor_node = valid_node_ids[1] + push!(schema_subgraph_nodes, anchor_node) + + # Phase B: Sequentially route paths to all other semantic coordinates + for target_node in valid_node_ids[2:end] + # Skip if an earlier loop trajectory already naturally absorbed this table + if target_node in schema_subgraph_nodes + continue + end + + # Calculate the shortest path tree from the CURRENT state of our subgraph + # We find the shortest path from the target back to ANY node currently in our tree + shortest_paths = dijkstra_shortest_paths(g, target_node) + + # Find which node currently in our subgraph is closest to the target node + closest_subgraph_node = 0 + min_distance = Inf + + for subgraph_node in schema_subgraph_nodes + dist = shortest_paths.dists[subgraph_node] + if dist < min_distance + min_distance = dist + closest_subgraph_node = subgraph_node + end + end + + # Reconstruct the path from the target node to the closest point on our existing tree + if closest_subgraph_node != 0 + curr = closest_subgraph_node + while curr != 0 + push!(schema_subgraph_nodes, curr) + curr = shortest_paths.parents[curr] + if curr == target_node + push!(schema_subgraph_nodes, target_node) + break + end + end + end + end + + # Map the unique structural nodes back to clean table names + return [id_to_table[node_id] for node_id in schema_subgraph_nodes] +end From d658d9a25bcaf5155a5492b52f2764650bf92c14 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 13 Jul 2026 21:22:38 +0700 Subject: [PATCH 02/10] update --- Manifest.toml | 80 ++++++++++++++++++++++++++--- Project.toml | 4 ++ src/llmUtil.jl | 135 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 205 insertions(+), 14 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 44f5eb7..3cee6c9 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "c825feef41198c770952e1181ec41e9f2aa0c3c0" +project_hash = "92ca9c293aa799cfd151fbc0abb978aa6e6ee00b" [[deps.Accessors]] deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] @@ -34,6 +34,12 @@ git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" version = "1.1.3" +[[deps.ArnoldiMethod]] +deps = ["LinearAlgebra", "Random", "StaticArrays"] +git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6" +uuid = "ec485272-7323-5ecc-a04f-4719b315124d" +version = "0.4.0" + [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" @@ -156,6 +162,20 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.12" + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + + [deps.Distances.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3" @@ -207,6 +227,7 @@ deps = ["LinearAlgebra"] git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" version = "1.16.0" +weakdeps = ["PDMats", "SparseArrays", "StaticArrays", "Statistics"] [deps.FillArrays.extensions] FillArraysPDMatsExt = "PDMats" @@ -214,23 +235,30 @@ version = "1.16.0" FillArraysStaticArraysExt = "StaticArrays" FillArraysStatisticsExt = "Statistics" - [deps.FillArrays.weakdeps] - PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" [[deps.GeneralUtils]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"] path = "." uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" version = "0.4.10" +[[deps.Graphs]] +deps = ["ArnoldiMethod", "DataStructures", "Inflate", "LinearAlgebra", "Random", "SimpleTraits", "SparseArrays", "Statistics"] +git-tree-sha1 = "7eb45fe833a5b7c51cf6d89c5a841d5967e44be3" +uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" +version = "1.14.0" + + [deps.Graphs.extensions] + GraphsSharedArraysExt = "SharedArrays" + + [deps.Graphs.weakdeps] + Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + [[deps.HTTP]] deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"] git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55" @@ -248,6 +276,11 @@ git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" version = "0.3.28" +[[deps.Inflate]] +git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d" +uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" +version = "0.1.5" + [[deps.InlineStrings]] git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" @@ -628,6 +661,12 @@ version = "1.4.10" uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "7ddb0b49c109481b046972c0e4ab02b2127d6a75" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.6" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" @@ -661,6 +700,25 @@ version = "2.8.0" [deps.SpecialFunctions.weakdeps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "246a8bb2e6667f832eea063c3a56aef96429a3db" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.18" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.4" + [[deps.Statistics]] deps = ["LinearAlgebra"] git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" @@ -697,6 +755,12 @@ version = "2.2.0" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" +[[deps.StringDistances]] +deps = ["Distances", "StatsAPI"] +git-tree-sha1 = "cd83a04baf746e3b43b83c61b7de77ab0409b80a" +uuid = "88034a9c-02f8-509d-84a9-84ec65e18404" +version = "1.0.0" + [[deps.StringManipulation]] deps = ["PrecompileTools"] git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5" diff --git a/Project.toml b/Project.toml index dd9233d..dad4a61 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" @@ -16,10 +17,13 @@ PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] +Graphs = "1.14.0" HTTP = "2.5.0 - 2.9.9" JSON = "1.3.0 - 1.9.9" NATS = "0.1.0" Revise = "3.13.2" +StringDistances = "1.0.0" diff --git a/src/llmUtil.jl b/src/llmUtil.jl index be57fba..b620cf7 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -1,9 +1,10 @@ module llmUtil -export formatLLMtext, extractthink, - checkAgentResponse_JSON, clean_json_response +export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response, + extract_vector_metadata, generate_embedding_payloads, resolve_semantic_cluster, + harvest_entity_catalog, resolve_entity -using UUIDs, JSON, Dates, DataFrames +using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs using GeneralUtils # ---------------------------------------------- 100 --------------------------------------------- # @@ -228,7 +229,7 @@ Returns a DataFrame designed for vector embedding generation. # Arguments - `pg_conn_str::String` - PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname") + PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret") # Return - `DataFrame` @@ -244,7 +245,7 @@ Returns a DataFrame designed for vector embedding generation. # Example ```julia julia> using GeneralUtils -julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb" +julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" julia> df = GeneralUtils.extract_vector_metadata(pg_conn) DataFrame 6 rows × 7 columns @@ -333,7 +334,7 @@ The function constructs rich text payloads by: # Example ```julia julia> using GeneralUtils -julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb" +julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" julia> df = GeneralUtils.extract_vector_metadata(pg_conn) julia> payloads = GeneralUtils.generate_embedding_payloads(df) 3-element Vector{Dict}: @@ -496,10 +497,132 @@ function resolve_semantic_cluster( end +""" Harvest entity catalog from database column. + +Extracts unique, non-null values from a specific column to build a local index for +semantic search or entity resolution. + +# Arguments +- `conn_str::String` + PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret") +- `table::String` + Table name to query +- `column::String` + Column name containing entity values + +# Return +- `Vector{String}` + A vector of unique, stripped strings from the specified column. Empty strings + are removed via `strip()`. + +# Details +The function: +1. Connects to PostgreSQL database +2. Executes `SELECT DISTINCT column FROM table WHERE column IS NOT NULL` +3. Converts result to DataFrame +4. Strips whitespace from each value and converts to String +5. Returns clean vector of unique entity values + +# Example +```julia +julia> using GeneralUtils +julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" +julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name") +["Apple", "Banana", "Orange", "Mango"] +``` +""" +function harvest_entity_catalog(conn_str::String, table::String, column::String)::Vector{String} + conn = LibPQ.Connection(conn_str) + + # We only care about unique, non-null values to keep the index fast and dense + query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" + + try + df = DataFrame(execute(conn, query)) + # Return as a clean array of strings + return String.(strip.(df[:, 1])) + finally + close(conn) + end +end +""" Resolve entity name from messy input using fuzzy string matching. +Matches user-provided text against a reference catalog using Jaro-Winkler similarity +and returns the closest matching exact string from the database catalog. +# Arguments +- `messy_input::String` + The user input text that may contain typos, compressed words, or variations. +- `catalog::Vector{String}` + A vector of valid, exact entity strings from the database. + +# Keyword Arguments +- `threshold::Float64` (default: `0.5`) + Minimum similarity score (0.0 to 1.0) required to return a match. Lower values + allow more lenient matching; higher values require closer matches. + +# Return +- `String` + The exact matching string from `catalog` if similarity score ≥ threshold, + otherwise an empty string `""`. + +# Details +The function: +1. Normalizes input to lowercase and strips whitespace +2. Computes Jaro-Winkler similarity score against each catalog entry +3. Applies substring fallback: if compressed words match (e.g., "HandOld" → "Hand Old Bar & Grill"), + boosts score to 0.85 +4. Returns the highest-scoring catalog entry if score ≥ threshold, else empty string + +# Example +```julia +julia> using GeneralUtils +julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"] +julia> GeneralUtils.resolve_entity("HandOld", catalog, threshold=0.5) +"Hand Old Bar & Grill" + +julia> GeneralUtils.resolve_entity("Wine Cellar", catalog, threshold=0.5) +"Wine Cellar" + +julia> GeneralUtils.resolve_entity("Unknown Place", catalog, threshold=0.5) +"" +``` +""" +function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String + best_match = "" + highest_score = 0.0 + + # Normalize input text to ensure case-insensitive matching + clean_input = lowercase(strip(messy_input)) + + for real_string in catalog + clean_real = lowercase(real_string) + + # Calculate phonetic/structural similarity score (0.0 to 1.0) + # JaroWinkler is optimized for short strings, names, and partial acronyms + score = compare(clean_real, clean_input, JaroWinkler()) + + # Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill" + # We strip spaces to check if the user just compressed words together + if contains(replace(clean_real, " " => ""), clean_input) + score = max(score, 0.85) + end + + if score > highest_score + highest_score = score + best_match = real_string + end + end + + # Only return if we cross our safety confidence barrier + if highest_score >= threshold + return best_match + end + + return "" # No confident match found +end From f28405f3f1db449bef476991717491d4e8f5b9b3 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 13 Jul 2026 21:23:21 +0700 Subject: [PATCH 03/10] update --- etc.jl | 355 --------------------------------------------------------- 1 file changed, 355 deletions(-) diff --git a/etc.jl b/etc.jl index 894b9a3..e69de29 100644 --- a/etc.jl +++ b/etc.jl @@ -1,355 +0,0 @@ - - -using LibPQ -using DataFrames - -""" - extract_vector_metadata(pg_conn_str::String) -> DataFrame - -Queries PostgreSQL system catalogs to extract a rich semantic text map of every -column in the database. Returns a DataFrame designed for vector embedding generation. -""" -function extract_vector_metadata(pg_conn_str::String) - conn = LibPQ.Connection(pg_conn_str) - - # This direct SQL query pulls the column specifications along with column-level descriptions - query = """ - SELECT - c.relname AS table_name, - a.attname AS column_name, - format_type(a.atttypid, a.atttypmod) AS data_type, - COALESCE(d.description, '') AS column_description, - CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, - CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key - FROM pg_attribute a - JOIN pg_class c ON c.oid = a.attrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - -- Join to fetch column comments/descriptions - LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum - -- Check if column is part of a Primary Key - LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid - AND pk.contype = 'p' - AND a.attnum = ANY(pk.conkey) - -- Check if column is part of a Foreign Key - LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid - AND fk.contype = 'f' - AND a.attnum = ANY(fk.conkey) - WHERE - n.nspname = 'public' -- Only user schemas - AND c.relkind = 'r' -- Only standard tables - AND a.attnum > 0 -- Skip system hidden columns - AND NOT a.attisdropped; -- Skip dropped columns - """ - - try - # Execute and format into a clean DataFrame - result = execute(conn, query) - df = DataFrame(result) - - # Create a unique document ID for each vector row - df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)] - - return df - finally - close(conn) - end -end - - - -""" - generate_embedding_payloads(df::DataFrame) -> Vector{Dict} - -Transforms the metadata DataFrame into structured text strings optimal for -vector space mapping. -""" -function generate_embedding_payloads(df::DataFrame) - payloads = Dict[] - - for row in eachrow(df) - # 1. Build a rich text description summarizing the column's role - text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)" - - if row.is_primary_key - text_payload *= " [PRIMARY KEY]" - end - if row.is_foreign_key - text_payload *= " [FOREIGN KEY RELATIONAL LINK]" - end - - # Append business descriptions if they exist in the database comments - if !isempty(strip(row.column_description)) - text_payload *= " | Description: $(row.column_description)" - else - text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture." - end - - # 2. Package everything neatly to be passed to your vector store client - push!(payloads, Dict( - "id" => row.vector_id, - "text_content" => text_payload, - "metadata" => Dict( - "table" => row.table_name, - "column" => row.column_name, - "type" => row.data_type - ) - )) - end - - return payloads -end - - -""" - resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String} - -Takes a scattered array of semantically matched tables from Stage 1, navigates -the undirected network structure, and isolates the minimum interconnected subgraph -required to weave ALL hits into a single valid SQL query. -""" -function resolve_semantic_cluster( - vector_hits::Vector{String}, - g::SimpleGraph, - table_to_id::Dict{String, Int}, - id_to_table::Dict{Int, String} - ) - # Filter out hits that don't exist in our actual database graph mapping - valid_node_ids = Int[] - for hit in vector_hits - if haskey(table_to_id, hit) - push!(valid_node_ids, table_to_id[hit]) - else - @warn "Vector hit '$hit' does not map to an existing database table." - end - end - - unique!(valid_node_ids) - - # Edge Case Handlers - if isempty(valid_node_ids) - return String[] - elseif length(valid_node_ids) == 1 - return [id_to_table[valid_node_ids[1]]] - end - - # The Isolated Subgraph Set to build our final context - schema_subgraph_nodes = Set{Int}() - - # Phase A: Select an initial anchor component. We use the highest-ranked vector hit. - anchor_node = valid_node_ids[1] - push!(schema_subgraph_nodes, anchor_node) - - # Phase B: Sequentially route paths to all other semantic coordinates - for target_node in valid_node_ids[2:end] - # Skip if an earlier loop trajectory already naturally absorbed this table - if target_node in schema_subgraph_nodes - continue - end - - # Calculate the shortest path tree from the CURRENT state of our subgraph - # We find the shortest path from the target back to ANY node currently in our tree - shortest_paths = dijkstra_shortest_paths(g, target_node) - - # Find which node currently in our subgraph is closest to the target node - closest_subgraph_node = 0 - min_distance = Inf - - for subgraph_node in schema_subgraph_nodes - dist = shortest_paths.dists[subgraph_node] - if dist < min_distance - min_distance = dist - closest_subgraph_node = subgraph_node - end - end - - # Reconstruct the path from the target node to the closest point on our existing tree - if closest_subgraph_node != 0 - curr = closest_subgraph_node - while curr != 0 - push!(schema_subgraph_nodes, curr) - curr = shortest_paths.parents[curr] - if curr == target_node - push!(schema_subgraph_nodes, target_node) - break - end - end - end - end - - # Map the unique structural nodes back to clean table names - return [id_to_table[node_id] for node_id in schema_subgraph_nodes] -end - -function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String}) - documents_dict = Dict("documents" => text) - payloads = [("documents", documents_dict, "dictionary")] - _, msg_envelope_json_str = msghandler.smartpack( - config["externalservice"]["servicesloadbalancer"]["nats"], - payloads; - msg_purpose="embedding", - broker_url=config["nats_server_info"]["url"], - fileserver_url=config["externalservice"]["fileserver"]["url"]) - - reply = NATS.request(nats_conn, - config["externalservice"]["servicesloadbalancer"]["nats"], - msg_envelope_json_str, timeout=120) - incoming_env_json_str = String(reply.payload) - incoming_env = msghandler.smartunpack(incoming_env_json_str) - embedding_response = incoming_env["payloads"][1][2] - - return embedding_response -end - -nats_conn = NATS.connect(config["nats_server_info"]["url"]) - -# Run the extractor -metadata_df = extract_vector_metadata(pg_conn_str) -embedding_ready = generate_embedding_payloads(metadata_df) - -println(embedding_ready[1]["text_content"]) -# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items." - -embedding_ready_2 = [i["text_content"] for i in embedding_ready] -table_embedding = get_embedding(nats_conn, embedding_ready_2) - -user_question = - """ - Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b} - """ -user_question_embedding = get_embedding(nats_conn, [user_question]) - - -using Distances -similarity = 1 - cosine_dist(Float64.(table_embedding["data"][1]["embedding"]), - Float64.(user_question_embedding["data"][1]["embedding"]) - ) -user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"]) -user_question_similarity = [] -for i in table_embedding["data"] - i_data = i["embedding"] - i_float = Float64.(i_data) - r = 1 - cosine_dist(i_float, user_question_embedding) - push!(user_question_similarity, r) -end - -new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) -sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min - -# top 20 of sorted_df get this tables -vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"] - -g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str) - -# tables that I should put schema in LLM context -optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) - - - -function related_tables_for_user_question() - - - - -end - -# ---------------------------------------------- 100 --------------------------------------------- # - - -# Agent 3 (The Entity Resolver): Instantly runs a fast, local token search (like BM25) to map messy user text (like HandOld) to the exact database string (Hand Old Bar & Grill) before the SQL is drafted. - -using StringDistances - -""" - harvest_entity_catalog(conn_str::String, table::String, column::String) -> Vector{String} - -Pulls unique, clean text strings from a specific entity column to build a local index. -""" -function harvest_entity_catalog(conn_str::String, table::String, column::String) - conn = LibPQ.Connection(conn_str) - - # We only care about unique, non-null values to keep the index fast and dense - query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" - - try - df = DataFrame(execute(conn, query)) - # Return as a clean array of strings - return String.(strip.(df[:, 1])) - finally - close(conn) - end -end - - -""" - resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.6) -> String - -Parses user text, matches it against the real database catalog, and returns -the exact string found in the database. Returns an empty string if no confident match. -""" -function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5) - best_match = "" - highest_score = 0.0 - - # Normalize input text to ensure case-insensitive matching - clean_input = lowercase(strip(messy_input)) - - for real_string in catalog - clean_real = lowercase(real_string) - - # Calculate phonetic/structural similarity score (0.0 to 1.0) - # JaroWinkler is optimized for short strings, names, and partial acronyms - score = compare(clean_real, clean_input, JaroWinkler()) - - # Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill" - # We strip spaces to check if the user just compressed words together - if contains(replace(clean_real, " " => ""), clean_input) - score = max(score, 0.85) - end - - if score > highest_score - highest_score = score - best_match = real_string - end - end - - # Only return if we cross our safety confidence barrier - if highest_score >= threshold - return best_match - end - - return "" # No confident match found -end - - - -winery_catalog = harvest_entity_catalog(conn_str, "wine", "winery") -# Let's assume the catalog contains: ["Hand Old Bar & Grill", "Bangkok Diner", "Phuket Seafood"] - -# 2. The user asks a messy question with a typo and compressed text -user_question = "What are the total sales at HandOld last week?" - -# 3. Agent 3 isolates potential nouns or scans the question against the index -# We look for words that don't match standard english dictionary tokens, or check the full string segments -detected_entity = "Jacob" - -# 4. Run the resolution engine -exact_db_string = resolve_entity(detected_entity, winery_catalog) -# "United States" - -println("Messy Input: ", detected_entity) -println("Resolved Engine Value: ", exact_db_string) -# Output: "Hand Old Bar & Grill" - - - - - - - - - - - - - - From a15630619a905c1890cf9542aac4b69488f6c4ce Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 14 Jul 2026 13:00:39 +0700 Subject: [PATCH 04/10] update --- Manifest.toml | 245 ++++++++++++++++++++++++++++++++++++++++++------- Project.toml | 2 + src/llmUtil.jl | 42 +++++---- 3 files changed, 237 insertions(+), 52 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 3cee6c9..03d6c68 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "92ca9c293aa799cfd151fbc0abb978aa6e6ee00b" +project_hash = "a1c572efdd97cec1f432040f02784b67635102aa" [[deps.Accessors]] deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] @@ -34,12 +34,22 @@ git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" version = "1.1.3" +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + [[deps.ArnoldiMethod]] deps = ["LinearAlgebra", "Random", "StaticArrays"] git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6" uuid = "ec485272-7323-5ecc-a04f-4719b315124d" version = "0.4.0" +[[deps.ArrowTypes]] +deps = ["Sockets", "UUIDs"] +git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101" +uuid = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +version = "2.3.0" + [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" @@ -53,6 +63,11 @@ git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643" uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" version = "1.2.2" +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + [[deps.CRC32c]] uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" version = "1.11.0" @@ -82,9 +97,9 @@ uuid = "944b1d66-785c-5afd-91f1-9de20f533193" version = "0.7.8" [[deps.CommonSolve]] -git-tree-sha1 = "99ee296f88c12485402e37c2fd025f95ae097637" +git-tree-sha1 = "eeaad7cef88554c2fa56b5a3f71cfd5cb708c662" uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" -version = "0.2.9" +version = "0.2.11" [[deps.Compat]] deps = ["TOML", "UUIDs"] @@ -135,6 +150,11 @@ git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" version = "4.1.1" +[[deps.DBInterface]] +git-tree-sha1 = "a444404b3f94deaa43ca2a58e18153a82695282b" +uuid = "a10d1c49-ce27-4219-8d33-6db1a4562965" +version = "2.6.1" + [[deps.DataAPI]] git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" @@ -162,6 +182,11 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" +[[deps.Decimals]] +git-tree-sha1 = "e98abef36d02a0ec385d68cd7dadbce9b28cbd88" +uuid = "abce61dc-4473-55a0-ba07-351d65e31d42" +version = "0.4.1" + [[deps.Distances]] deps = ["LinearAlgebra", "Statistics", "StatsAPI"] git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" @@ -176,6 +201,11 @@ version = "0.10.12" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3" @@ -199,25 +229,32 @@ git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.5" +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + [[deps.EnumX]] git-tree-sha1 = "c49898e8438c828577f04b92fc9368c388ac783c" uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" version = "1.0.7" +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + [[deps.FilePathsBase]] deps = ["Compat", "Dates"] git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" uuid = "48062228-2e41-5def-b9a4-89aafe57970f" version = "0.9.24" +weakdeps = ["Mmap", "Test"] [deps.FilePathsBase.extensions] FilePathsBaseMmapExt = "Mmap" FilePathsBaseTestExt = "Test" - [deps.FilePathsBase.weakdeps] - Mmap = "a63ad114-7e13-5084-954f-fe012c677804" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" @@ -240,8 +277,13 @@ deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" +[[deps.Gamma]] +git-tree-sha1 = "86f86b6168a016ed88e4ae4e64577b98c3b59e8e" +uuid = "a0844989-3bd2-4988-8bea-c9407ab0941b" +version = "1.1.0" + [[deps.GeneralUtils]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "Graphs", "HTTP", "JSON", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"] path = "." uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" version = "0.4.10" @@ -261,9 +303,9 @@ version = "1.14.0" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"] -git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55" +git-tree-sha1 = "c2c808326222b6dc4bec295a83b55f79aeec98e0" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "2.5.0" +version = "2.5.5" [[deps.HashArrayMappedTries]] git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" @@ -271,10 +313,22 @@ uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" version = "0.2.0" [[deps.HypergeometricFunctions]] -deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] -git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" +deps = ["Gamma", "LinearAlgebra"] +git-tree-sha1 = "18d7deab5fb0440dc6a7b6993c5c27b25420de10" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" -version = "0.3.28" +version = "0.3.29" + +[[deps.ICU_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b3d8be712fbf9237935bde0ce9b5a736ae38fc34" +uuid = "a51ab1cf-af8e-5615-a023-bc2c838bba6b" +version = "76.2.0+0" + +[[deps.Infinity]] +deps = ["Dates", "Random", "Requires"] +git-tree-sha1 = "cf8234411cbeb98676c173f930951ea29dca3b23" +uuid = "a303e19e-6eb4-11e9-3b09-cd9505f79100" +version = "0.2.4" [[deps.Inflate]] git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d" @@ -285,33 +339,33 @@ version = "0.1.5" git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" version = "1.4.5" +weakdeps = ["ArrowTypes", "Parsers"] [deps.InlineStrings.extensions] ArrowTypesExt = "ArrowTypes" ParsersExt = "Parsers" - [deps.InlineStrings.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" - [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" +[[deps.Intervals]] +deps = ["ArrowTypes", "Dates", "Printf", "RecipesBase", "Serialization", "TimeZones"] +git-tree-sha1 = "d6fe00b123e32ddd17231b35d69a6394e696fd5a" +uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5" +version = "1.11.0" + [[deps.InverseFunctions]] git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb" uuid = "3587e190-3f89-42d0-90ee-14403ec27112" version = "0.1.17" +weakdeps = ["Dates", "Test"] [deps.InverseFunctions.extensions] InverseFunctionsDatesExt = "Dates" InverseFunctionsTestExt = "Test" - [deps.InverseFunctions.weakdeps] - Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - [[deps.InvertedIndices]] git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" @@ -322,6 +376,11 @@ git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" version = "0.2.6" +[[deps.IterTools]] +git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.10.0" + [[deps.IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" @@ -338,25 +397,21 @@ deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs" git-tree-sha1 = "c89d196f5ffb64bfbf80985b699ea913b0d2c211" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "1.6.1" +weakdeps = ["ArrowTypes"] [deps.JSON.extensions] JSONArrowExt = ["ArrowTypes"] - [deps.JSON.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" version = "1.14.3" +weakdeps = ["ArrowTypes"] [deps.JSON3.extensions] JSON3ArrowExt = ["ArrowTypes"] - [deps.JSON3.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - [[deps.JuliaInterpreter]] deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] git-tree-sha1 = "58927c485919bf17ea308d9d82156de1adf4b006" @@ -368,11 +423,32 @@ deps = ["StyledStrings"] uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" version = "1.12.0" +[[deps.Kerberos_krb5_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "0f2899fdadaab4b8f57db558ba21bdb4fb52f1f0" +uuid = "b39eb1a6-c29a-53d7-8c32-632cd16f18da" +version = "1.21.3+0" + [[deps.LaTeXStrings]] git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.4.0" +[[deps.LayerDicts]] +git-tree-sha1 = "6087ad3521d6278ebe5c27ae55e7bbb15ca312cb" +uuid = "6f188dcb-512c-564b-bc01-e0f76e72f166" +version = "1.0.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -383,6 +459,18 @@ deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" version = "1.9.0+0" +[[deps.LibPQ]] +deps = ["CEnum", "DBInterface", "Dates", "Decimals", "DocStringExtensions", "FileWatching", "Infinity", "Intervals", "IterTools", "LayerDicts", "LibPQ_jll", "Libdl", "Memento", "OffsetArrays", "SQLStrings", "Tables", "TimeZones", "UTCDateTimes"] +git-tree-sha1 = "3d227cd13cbf1e9a54d7748dab33e078da6f9168" +uuid = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1" +version = "1.18.0" + +[[deps.LibPQ_jll]] +deps = ["Artifacts", "ICU_jll", "JLLWrappers", "Kerberos_krb5_jll", "Libdl", "OpenSSL_jll", "Zstd_jll"] +git-tree-sha1 = "c692057e05ba6da348bc45d5dab8c7a2c88da518" +uuid = "08be9ffa-1c94-5ee5-a977-46a84ec9b350" +version = "16.14.0+0" + [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "OpenSSL_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" @@ -445,6 +533,12 @@ git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8" uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" version = "2.28.1010+0" +[[deps.Memento]] +deps = ["Dates", "Distributed", "Requires", "Serialization", "Sockets", "Test", "UUIDs"] +git-tree-sha1 = "e03a25cb3b6569623f8246d3d8b3faa7ce86f4ad" +uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" +version = "1.5.0" + [[deps.Missings]] deps = ["DataAPI"] git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" @@ -455,6 +549,12 @@ version = "1.2.0" uuid = "a63ad114-7e13-5084-954f-fe012c677804" version = "1.11.0" +[[deps.Mocking]] +deps = ["Compat", "ExprTools"] +git-tree-sha1 = "2c140d60d7cb82badf06d8783800d0bcd1a7daa2" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.8.1" + [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" version = "2025.11.4" @@ -475,6 +575,17 @@ version = "1.0.3" uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" +[[deps.OffsetArrays]] +git-tree-sha1 = "117432e406b5c023f665fa73dc26e79ec3630151" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.17.0" + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + + [deps.OffsetArrays.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" @@ -542,15 +653,17 @@ version = "0.4.2" [[deps.PrettyTables]] deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "624de6279ab7d94fc9f672f0068107eb6619732c" +git-tree-sha1 = "ebf455bb866ee6737030e3d3816bb6a0683c4325" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "3.3.2" +version = "3.4.0" [deps.PrettyTables.extensions] + PrettyTablesExcelExt = "XLSX" PrettyTablesTypstryExt = "Typstry" [deps.PrettyTables.weakdeps] Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e" + XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0" [[deps.Printf]] deps = ["Unicode"] @@ -584,11 +697,23 @@ deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" version = "1.11.0" +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "1.2.2" +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + [[deps.Reseau]] deps = ["NetworkOptions", "OpenSSL_jll", "PrecompileTools", "Random", "SHA"] git-tree-sha1 = "0eab6d95ed40c2ef3992255c1c71e4f9748932b5" @@ -600,13 +725,11 @@ deps = ["CRC32c", "CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInte git-tree-sha1 = "27e3ee13fc8739a59b380d6163d6a82f52c03bd7" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" version = "3.15.1" +weakdeps = ["Distributed"] [deps.Revise.extensions] DistributedExt = "Distributed" - [deps.Revise.weakdeps] - Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" - [[deps.Rmath]] deps = ["Random", "Rmath_jll"] git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" @@ -621,9 +744,9 @@ version = "0.5.1+0" [[deps.Roots]] deps = ["Accessors", "CommonSolve", "Printf"] -git-tree-sha1 = "91cfb1cb4f6e27557cc2df798a31eff6089a41eb" +git-tree-sha1 = "125cbd31a56de53169c3eed9c17180bc6c245f83" uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" -version = "3.0.0" +version = "3.0.4" [deps.Roots.extensions] RootsChainRulesCoreExt = "ChainRulesCore" @@ -645,12 +768,23 @@ version = "3.0.0" uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" +[[deps.SQLStrings]] +git-tree-sha1 = "55de0530689832b1d3d43491ee6b67bd54d3323c" +uuid = "af517c2e-c243-48fa-aab8-efac3db270f5" +version = "0.1.0" + [[deps.ScopedValues]] deps = ["HashArrayMappedTries", "Logging"] git-tree-sha1 = "67a144433c4ce877ee6d1ada69a124d6b1ecf7be" uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" version = "1.6.2" +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.3.0" + [[deps.SentinelArrays]] deps = ["Dates", "Random"] git-tree-sha1 = "084c47c7c5ce5cfecefa0a98dff69eb3646b5a80" @@ -807,6 +941,12 @@ deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" version = "1.0.3" +[[deps.TZJData]] +deps = ["Artifacts"] +git-tree-sha1 = "72df96b3a595b7aab1e101eb07d2a435963a97e2" +uuid = "dc5dba14-91b3-4cab-a142-028a31da12f7" +version = "1.5.0+2025b" + [[deps.TableTraits]] deps = ["IteratorInterfaceExtensions"] git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" @@ -819,6 +959,21 @@ git-tree-sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" version = "1.13.0" +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimeZones]] +deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"] +git-tree-sha1 = "d422301b2a1e294e3e4214061e44f338cafe18a2" +uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" +version = "1.22.2" +weakdeps = ["RecipesBase"] + + [deps.TimeZones.extensions] + TimeZonesRecipesBaseExt = "RecipesBase" + [[deps.TranscodingStreams]] git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" @@ -829,6 +984,12 @@ git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" version = "1.6.1" +[[deps.UTCDateTimes]] +deps = ["Dates", "TimeZones"] +git-tree-sha1 = "4af3552bf0cf4a071bf3d14bd20023ea70f31b62" +uuid = "0f7cfa37-7abf-4834-b969-a8aa512401c2" +version = "1.6.1" + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -854,6 +1015,12 @@ deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.3.1+2" +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" @@ -864,3 +1031,13 @@ deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" version = "1.0.21+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/Project.toml b/Project.toml index dad4a61..1f2c018 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1" NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -24,6 +25,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" Graphs = "1.14.0" HTTP = "2.5.0 - 2.9.9" JSON = "1.3.0 - 1.9.9" +LibPQ = "1.18.0" NATS = "0.1.0" Revise = "3.13.2" StringDistances = "1.0.0" diff --git a/src/llmUtil.jl b/src/llmUtil.jl index b620cf7..44516f5 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -4,8 +4,8 @@ export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response extract_vector_metadata, generate_embedding_payloads, resolve_semantic_cluster, harvest_entity_catalog, resolve_entity -using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs -using GeneralUtils +using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ +using ..util # ---------------------------------------------- 100 --------------------------------------------- # @@ -294,7 +294,7 @@ function extract_vector_metadata(pg_conn_str::String)::DataFrame try # Execute and format into a clean DataFrame - result = execute(conn, query) + result = LibPQ.execute(conn, query) df = DataFrame(result) # Create a unique document ID for each vector row @@ -532,18 +532,24 @@ julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_nam ``` """ function harvest_entity_catalog(conn_str::String, table::String, column::String)::Vector{String} - conn = LibPQ.Connection(conn_str) - - # We only care about unique, non-null values to keep the index fast and dense - query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" - - try - df = DataFrame(execute(conn, query)) - # Return as a clean array of strings - return String.(strip.(df[:, 1])) - finally - close(conn) - end + conn = LibPQ.Connection(conn_str) + return harvest_entity_catalog(conn, table, column) +end + +function harvest_entity_catalog(conn, table::String, column::String)::Vector{String} + + # We only care about unique, non-null values to keep the index fast and dense + query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" + + try + df = DataFrame(LibPQ.execute(conn, query)) + # Return as a clean array of strings + return String.(strip.(df[:, 1])) + catch + return String[] + finally + close(conn) + end end @@ -580,13 +586,13 @@ The function: ```julia julia> using GeneralUtils julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"] -julia> GeneralUtils.resolve_entity("HandOld", catalog, threshold=0.5) +julia> GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.5) "Hand Old Bar & Grill" -julia> GeneralUtils.resolve_entity("Wine Cellar", catalog, threshold=0.5) +julia> GeneralUtils.resolve_entity("Wine Cellar", catalog; threshold=0.5) "Wine Cellar" -julia> GeneralUtils.resolve_entity("Unknown Place", catalog, threshold=0.5) +julia> GeneralUtils.resolve_entity("Unknown Place", catalog; threshold=0.5) "" ``` """ From 95954249ce15ff7df9b58e1a19627cf7c861135c Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 14 Jul 2026 18:13:31 +0700 Subject: [PATCH 05/10] update --- src/llmUtil.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/llmUtil.jl b/src/llmUtil.jl index 44516f5..e0cbef0 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -1,7 +1,7 @@ module llmUtil export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response, - extract_vector_metadata, generate_embedding_payloads, resolve_semantic_cluster, + extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster, harvest_entity_catalog, resolve_entity using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ @@ -246,7 +246,7 @@ Returns a DataFrame designed for vector embedding generation. ```julia julia> using GeneralUtils julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" -julia> df = GeneralUtils.extract_vector_metadata(pg_conn) +julia> df = GeneralUtils.extract_column_metadata(pg_conn) DataFrame 6 rows × 7 columns table_name column_name data_type column_description is_primary_key is_foreign_key vector_id @@ -259,7 +259,7 @@ products price numeric Product price false false products user_id integer Reference to user false true col_products_user_id ``` """ -function extract_vector_metadata(pg_conn_str::String)::DataFrame +function extract_column_metadata(pg_conn_str::String)::DataFrame conn = LibPQ.Connection(pg_conn_str) # This direct SQL query pulls the column specifications along with column-level descriptions @@ -309,12 +309,12 @@ end """ Generate embedding payloads from vector metadata DataFrame. -Transforms the metadata DataFrame from `extract_vector_metadata` into a vector of +Transforms the metadata DataFrame from `extract_column_metadata` into a vector of dictionaries structured for vector embedding storage and retrieval. # Arguments - `df::DataFrame` - A DataFrame with columns from `extract_vector_metadata`: `table_name`, `column_name`, + A DataFrame with columns from `extract_column_metadata`: `table_name`, `column_name`, `data_type`, `column_description`, `is_primary_key`, `is_foreign_key`, `vector_id`. # Return @@ -335,7 +335,7 @@ The function constructs rich text payloads by: ```julia julia> using GeneralUtils julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" -julia> df = GeneralUtils.extract_vector_metadata(pg_conn) +julia> df = GeneralUtils.extract_column_metadata(pg_conn) julia> payloads = GeneralUtils.generate_embedding_payloads(df) 3-element Vector{Dict}: Dict("id" => "col_users_id", "text_content" => "Table: users | Column: id | Type: integer [PRIMARY KEY] | Description: User ID", "metadata" => Dict("table" => "users", "column" => "id", "type" => "integer")) From 1ad46c6e18d0d812795a30fcd03e7a0a3c7ffbca Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 15 Jul 2026 07:11:09 +0700 Subject: [PATCH 06/10] update --- etc.jl | 293 +++++++++++++++++++++++++++++++++++++++++++++++++ src/llmUtil.jl | 159 +++++++++++++++++++++------ 2 files changed, 419 insertions(+), 33 deletions(-) diff --git a/etc.jl b/etc.jl index e69de29..570b067 100644 --- a/etc.jl +++ b/etc.jl @@ -0,0 +1,293 @@ + + + +using LibPQ, JSON, Graphs, DataFrames +config = JSON.parsefile("./appconfig.json") +host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':') +port = parse(Int, _port) +dbname = "winedb" +user = config["externalservice"]["sommpanion_db"]["user"] +password = config["externalservice"]["sommpanion_db"]["password"] +pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password" +db_connection = LibPQ.Connection(pg_conn_str) + + + + + +# ---------------------------------------------- 100 --------------------------------------------- # + + +using LibPQ +using DataFrames + +""" + extract_column_metadata(pg_conn_str::String) -> DataFrame + +Queries PostgreSQL system catalogs to extract a rich semantic text map of every +column in the database. Returns a DataFrame designed for vector embedding generation. +""" +function extract_column_metadata(pg_conn_str::String) + conn = LibPQ.Connection(pg_conn_str) + + # This direct SQL query pulls the column specifications along with column-level descriptions + query = """ + SELECT + c.relname AS table_name, + a.attname AS column_name, + format_type(a.atttypid, a.atttypmod) AS data_type, + COALESCE(d.description, '') AS column_description, + CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, + CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + -- Join to fetch column comments/descriptions + LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum + -- Check if column is part of a Primary Key + LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid + AND pk.contype = 'p' + AND a.attnum = ANY(pk.conkey) + -- Check if column is part of a Foreign Key + LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid + AND fk.contype = 'f' + AND a.attnum = ANY(fk.conkey) + WHERE + n.nspname = 'public' -- Only user schemas + AND c.relkind = 'r' -- Only standard tables + AND a.attnum > 0 -- Skip system hidden columns + AND NOT a.attisdropped; -- Skip dropped columns + """ + + try + # Execute and format into a clean DataFrame + result = execute(conn, query) + df = DataFrame(result) + + # Create a unique document ID for each vector row + df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)] + + return df + finally + close(conn) + end +end + + + +""" + generate_embedding_payloads(df::DataFrame) -> Vector{Dict} + +Transforms the metadata DataFrame into structured text strings optimal for +vector space mapping. +""" +function generate_embedding_payloads(df::DataFrame) + payloads = Dict[] + + for row in eachrow(df) + # 1. Build a rich text description summarizing the column's role + text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)" + + if row.is_primary_key + text_payload *= " [PRIMARY KEY]" + end + if row.is_foreign_key + text_payload *= " [FOREIGN KEY RELATIONAL LINK]" + end + + # Append business descriptions if they exist in the database comments + if !isempty(strip(row.column_description)) + text_payload *= " | Description: $(row.column_description)" + else + text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture." + end + + # 2. Package everything neatly to be passed to your vector store client + push!(payloads, Dict( + "id" => row.vector_id, + "text_content" => text_payload, + "metadata" => Dict( + "table" => row.table_name, + "column" => row.column_name, + "type" => row.data_type + ) + )) + end + + return payloads +end + + +""" + resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String} + +Takes a scattered array of semantically matched tables from Stage 1, navigates +the undirected network structure, and isolates the minimum interconnected subgraph +required to weave ALL hits into a single valid SQL query. +""" +function resolve_semantic_cluster( + vector_hits::Vector{String}, + g::SimpleGraph, + table_to_id::Dict{String, Int}, + id_to_table::Dict{Int, String} +) + # Filter out hits that don't exist in our actual database graph mapping + valid_node_ids = Int[] + for hit in vector_hits + if haskey(table_to_id, hit) + push!(valid_node_ids, table_to_id[hit]) + else + @warn "Vector hit '$hit' does not map to an existing database table." + end + end + + unique!(valid_node_ids) + + # Edge Case Handlers + if isempty(valid_node_ids) + return String[] + elseif length(valid_node_ids) == 1 + return [id_to_table[valid_node_ids[1]]] + end + + # The Isolated Subgraph Set to build our final context + schema_subgraph_nodes = Set{Int}() + + # Phase A: Select an initial anchor component. We use the highest-ranked vector hit. + anchor_node = valid_node_ids[1] + push!(schema_subgraph_nodes, anchor_node) + + # Phase B: Sequentially route paths to all other semantic coordinates + for target_node in valid_node_ids[2:end] + # Skip if an earlier loop trajectory already naturally absorbed this table + if target_node in schema_subgraph_nodes + continue + end + + # Calculate the shortest path tree from the CURRENT state of our subgraph + # We find the shortest path from the target back to ANY node currently in our tree + shortest_paths = dijkstra_shortest_paths(g, target_node) + + # Find which node currently in our subgraph is closest to the target node + closest_subgraph_node = 0 + min_distance = Inf + + for subgraph_node in schema_subgraph_nodes + dist = shortest_paths.dists[subgraph_node] + if dist < min_distance + min_distance = dist + closest_subgraph_node = subgraph_node + end + end + + # Reconstruct the path from the target node to the closest point on our existing tree + if closest_subgraph_node != 0 + curr = closest_subgraph_node + while curr != 0 + push!(schema_subgraph_nodes, curr) + curr = shortest_paths.parents[curr] + if curr == target_node + push!(schema_subgraph_nodes, target_node) + break + end + end + end + end + + # Map the unique structural nodes back to clean table names + return [id_to_table[node_id] for node_id in schema_subgraph_nodes] +end + +function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String}) + documents_dict = Dict("documents" => text) + payloads = [("documents", documents_dict, "dictionary")] + _, msg_envelope_json_str = msghandler.smartpack( + config["externalservice"]["servicesloadbalancer"]["nats"], + payloads; + msg_purpose="embedding", + broker_url=config["nats_server_info"]["url"], + fileserver_url=config["externalservice"]["fileserver"]["url"]) + + reply = NATS.request(nats_conn, + config["externalservice"]["servicesloadbalancer"]["nats"], + msg_envelope_json_str, timeout=120) + incoming_env_json_str = String(reply.payload) + incoming_env = msghandler.smartunpack(incoming_env_json_str) + embedding_response = incoming_env["payloads"][1][2] + + return embedding_response +end + +nats_conn = NATS.connect(config["nats_server_info"]["url"]) + +# Run the extractor +metadata_df = extract_column_metadata(pg_conn_str) +embedding_ready = generate_embedding_payloads(metadata_df) + +println(embedding_ready[1]["text_content"]) +# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items." + +# use only text content +embedding_ready_2 = [i["text_content"] for i in embedding_ready] +table_embedding = get_embedding(nats_conn, embedding_ready_2) + +user_question = + """ + Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b} + """ +user_question_embedding = get_embedding(nats_conn, [user_question]) + + +using Distances +# similarity = 1 - Distances.cosine_dist(Float64.(table_embedding["data"][1]["embedding"]), +# Float64.(user_question_embedding["data"][1]["embedding"]) +# ) +user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"]) +user_question_similarity = [] +for i in table_embedding["data"] + i_data = i["embedding"] + i_float = Float64.(i_data) + r = 1 - Distances.cosine_dist(i_float, user_question_embedding) + push!(user_question_similarity, r) +end + +new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) +sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min + +# top 20 of sorted_df get this tables +vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"] + +g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str) + +# tables that I should put schema in LLM context +optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) + + + +function find_related_tables_for_user_question(question::String) + metadata_df = extract_column_metadata(pg_conn_str) + embedding_ready = generate_embedding_payloads(metadata_df) + + # use only text content + embedding_ready_2 = [i["text_content"] for i in embedding_ready] + table_embedding = get_embedding(nats_conn, embedding_ready_2) + user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"]) + user_question_similarity = [] + for i in table_embedding["data"] + i_data = i["embedding"] + i_float = Float64.(i_data) + r = 1 - Distances.cosine_dist(i_float, user_question_embedding) + push!(user_question_similarity, r) + end + + new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) + sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min + + #WORKING extract top 20 rows of sorted_df to get tables that related to user question + vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"] + + g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str) + + # tables that I should put schema in LLM context + return optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) +end diff --git a/src/llmUtil.jl b/src/llmUtil.jl index e0cbef0..87658d4 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -177,6 +177,7 @@ function checkAgentResponse_JSON(responsedict::T1, requiredKeys::T2 return (ispass, errormsg) end + """ Convert a plain text string containing key-value pairs into a JSON-formatted string. This function takes text containing key-value pairs (typically extracted from LLM responses) @@ -221,6 +222,96 @@ function clean_json_response(text::String) end + +""" + harvest_db_undirected_schema_graph(pg_conn_str::String) + +Connects to a PostgreSQL instance, queries its metadata catalogs, and returns: +1. `g::SimpleDiGraph`: The structural graph where nodes are tables. +2. `id_to_table::Dict{Int, String}`: Maps numerical node IDs to real table names. +3. `table_to_id::Dict{String, Int}`: Maps table names back to graph node IDs. +""" +function harvest_db_undirected_schema_graph(pg_conn_str) + conn = LibPQ.Connection(pg_conn_str) + try + # 1. Fetch all user tables + table_query = """ + SELECT c.relname AS table_name, c.oid + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind = 'r'; + """ + table_df = DataFrame(execute(conn, table_query)) + table_names = table_df.table_name + + num_tables = length(table_names) + table_to_id = Dict{String, Int}(name => i for (i, name) in enumerate(table_names)) + id_to_table = Dict{Int, String}(i => name for (i, name) in enumerate(table_names)) + + # CRITICAL: Use SimpleGraph (Undirected) so pathfinding can traverse both ways + g = SimpleGraph(num_tables) + + # 2. Strategy A: Extract Explicit Foreign Keys + fk_query = """ + SELECT + conrelid::regclass::text AS source_table, + confrelid::regclass::text AS target_table + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE c.contype = 'f' AND n.nspname = 'public'; + """ + fk_df = DataFrame(execute(conn, fk_query)) + + for row in eachrow(fk_df) + src = split(replace(row.source_table, "\"" => ""), '.')[end] + tgt = split(replace(row.target_table, "\"" => ""), '.')[end] + + if haskey(table_to_id, src) && haskey(table_to_id, tgt) + add_edge!(g, table_to_id[src], table_to_id[tgt]) + end + end + + # 3. Strategy B: Fallback Name Matching (For missing explicit constraints) + # Fetch all columns for all tables + col_query = """ + SELECT a.attname AS column_name, c.relname AS table_name + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped; + """ + col_df = DataFrame(execute(conn, col_query)) + + # Group columns by name to see who shares what fields + for gdf in groupby(col_df, :column_name) + col_name = gdf.column_name[1] + + # We only infer connections on ID fields (e.g., seller_id, product_id) + # Avoid matching generic names like 'id', 'created_at', or 'name' + if endswith(lowercase(col_name), "_id") && col_name != "id" + sharing_tables = gdf.table_name + + # Connect all tables that share this ID column + for i in 1:length(sharing_tables), j in (i+1):length(sharing_tables) + t1 = sharing_tables[i] + t2 = sharing_tables[j] + + if haskey(table_to_id, t1) && haskey(table_to_id, t2) + add_edge!(g, table_to_id[t1], table_to_id[t2]) + end + end + end + end + + return g, id_to_table, table_to_id + + finally + close(conn) + end +end + + + """ Extract vector metadata from PostgreSQL database. Queries PostgreSQL system catalogs to extract column metadata including table names, @@ -261,36 +352,38 @@ products user_id integer Reference to user false true """ function extract_column_metadata(pg_conn_str::String)::DataFrame conn = LibPQ.Connection(pg_conn_str) - + return extract_column_metadata(conn) +end + +function extract_column_metadata(conn::LibPQ.Connection)::DataFrame # This direct SQL query pulls the column specifications along with column-level descriptions - query = - """ - SELECT - c.relname AS column_name, - format_type(a.atttypid, a.atttypmod) AS data_type, - COALESCE(d.description, '') AS column_description, - CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, - CASE WHEN AS table_name, - a.attnamefk.contype = 'f' THEN true ELSE false END AS is_foreign_key - FROM pg_attribute a - JOIN pg_class c ON c.oid = a.attrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - -- Join to fetch column comments/descriptions - LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum - -- Check if column is part of a Primary Key - LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid - AND pk.contype = 'p' - AND a.attnum = ANY(pk.conkey) - -- Check if column is part of a Foreign Key - LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid - AND fk.contype = 'f' - AND a.attnum = ANY(fk.conkey) - WHERE - n.nspname = 'public' -- Only user schemas - AND c.relkind = 'r' -- Only standard tables - AND a.attnum > 0 -- Skip system hidden columns - AND NOT a.attisdropped; -- Skip dropped columns - """ + query = """ + SELECT + c.relname AS table_name, + a.attname AS column_name, + format_type(a.atttypid, a.atttypmod) AS data_type, + COALESCE(d.description, '') AS column_description, + CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, + CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + -- Join to fetch column comments/descriptions + LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum + -- Check if column is part of a Primary Key + LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid + AND pk.contype = 'p' + AND a.attnum = ANY(pk.conkey) + -- Check if column is part of a Foreign Key + LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid + AND fk.contype = 'f' + AND a.attnum = ANY(fk.conkey) + WHERE + n.nspname = 'public' -- Only user schemas + AND c.relkind = 'r' -- Only standard tables + AND a.attnum > 0 -- Skip system hidden columns + AND NOT a.attisdropped; -- Skip dropped columns + """ try # Execute and format into a clean DataFrame @@ -503,7 +596,7 @@ Extracts unique, non-null values from a specific column to build a local index f semantic search or entity resolution. # Arguments -- `conn_str::String` +- `pg_conn_str::String` PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret") - `table::String` Table name to query @@ -531,12 +624,12 @@ julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_nam ["Apple", "Banana", "Orange", "Mango"] ``` """ -function harvest_entity_catalog(conn_str::String, table::String, column::String)::Vector{String} - conn = LibPQ.Connection(conn_str) +function harvest_entity_catalog(pg_conn_str::String, table::String, column::String)::Vector{String} + conn = LibPQ.Connection(pg_conn_str) return harvest_entity_catalog(conn, table, column) end -function harvest_entity_catalog(conn, table::String, column::String)::Vector{String} +function harvest_entity_catalog(conn::LibPQ.Connection, table::String, column::String)::Vector{String} # We only care about unique, non-null values to keep the index fast and dense query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" From 73ec3bbb0432154d69f831d3d2959bc7dad59be0 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 15 Jul 2026 07:19:01 +0700 Subject: [PATCH 07/10] update --- src/llmUtil.jl | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/llmUtil.jl b/src/llmUtil.jl index 87658d4..5545822 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -2,7 +2,7 @@ module llmUtil export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response, extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster, - harvest_entity_catalog, resolve_entity + harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ using ..util @@ -223,13 +223,35 @@ end -""" - harvest_db_undirected_schema_graph(pg_conn_str::String) +""" Harvest database schema as undirected graph from PostgreSQL. -Connects to a PostgreSQL instance, queries its metadata catalogs, and returns: -1. `g::SimpleDiGraph`: The structural graph where nodes are tables. -2. `id_to_table::Dict{Int, String}`: Maps numerical node IDs to real table names. -3. `table_to_id::Dict{String, Int}`: Maps table names back to graph node IDs. +Extracts table structure and relationships from a PostgreSQL database by querying +system catalogs to build a graph representation of tables and their relationships. + +# Arguments +- `pg_conn_str::String` + PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret") + +# Return +- `g::SimpleGraph`: An undirected graph where nodes represent tables and edges represent + relationships (foreign keys or shared ID column patterns). +- `id_to_table::Dict{Int, String}`: Maps numerical node IDs (1..n) to actual table names. +- `table_to_id::Dict{String, Int}`: Reverse mapping from table names to graph node IDs. + +# Details +The function extracts schema information using two strategies: +1. **Explicit Foreign Keys**: Queries `pg_constraint` for actual foreign key relationships +2. **Fallback Name Matching**: Infers relationships from shared column naming patterns + (e.g., `seller_id`, `product_id` columns across tables) + +# Example +```julia +julia> using GeneralUtils +julia> pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret" +julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str) +julia> vertices(g) +10 +``` """ function harvest_db_undirected_schema_graph(pg_conn_str) conn = LibPQ.Connection(pg_conn_str) @@ -292,7 +314,7 @@ function harvest_db_undirected_schema_graph(pg_conn_str) sharing_tables = gdf.table_name # Connect all tables that share this ID column - for i in 1:length(sharing_tables), j in (i+1):length(sharing_tables) + for i in eachindex(sharing_tables), j in (i+1):length(sharing_tables) t1 = sharing_tables[i] t2 = sharing_tables[j] From 55cc0f78c93176619e876946cf9d585657473f8c Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 15 Jul 2026 07:49:21 +0700 Subject: [PATCH 08/10] update --- src/llmUtil.jl | 122 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/src/llmUtil.jl b/src/llmUtil.jl index 5545822..f2c603b 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -252,6 +252,45 @@ julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_g julia> vertices(g) 10 ``` + +# Integration Guide: Finding Related Tables for User Questions +This function is designed to work together with `extract_column_metadata` and `resolve_semantic_cluster` to answer user questions by identifying related tables: + +```julia +# Step 1: Extract column metadata and generate embeddings for semantic search +pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret" +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) + +# Use only text content for embedding +embedding_ready_2 = [i["text_content"] for i in embedding_ready] +table_embedding = get_embedding(embedding_ready_2) + +# Get embedding for user question +_user_question_embedding = get_embedding([question]) +user_question_embedding = Float64.(_user_question_embedding["data"][1]["embedding"]) + +# Calculate similarity between question and all columns +user_question_similarity = [] +for i in table_embedding["data"] + i_data = i["embedding"] + i_float = Float64.(i_data) + r = 1 - Distances.cosine_dist(i_float, user_question_embedding) + push!(user_question_similarity, r) +end + +# Step 2: Find top related tables +new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) +sorted_df = sort(new_df, :user_question_similarity, rev=true) +_top_20_tables = unique(sorted_df[1:20, :table_name]) +top_20_tables = [i for i in _top_20_tables] + +# Step 3: Build schema graph and resolve table relationships +g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str) +table_relationship = GeneralUtils.resolve_semantic_cluster(top_20_tables, g, table_to_id, id_to_table) + +# table_relationship now contains tables in the order they should be joined +``` """ function harvest_db_undirected_schema_graph(pg_conn_str) conn = LibPQ.Connection(pg_conn_str) @@ -371,6 +410,21 @@ products id integer Product ID true false products price numeric Product price false false col_products_price products user_id integer Reference to user false true col_products_user_id ``` + +# Integration Guide +This function is the first step in the semantic table discovery pipeline. Use it with `generate_embedding_payloads` to prepare data for vector similarity search: + +```julia +# Step 1: Extract metadata +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) + +# Step 2: Generate embedding payloads for semantic search +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) + +# Use text_content field for embedding +text_content_list = [i["text_content"] for i in embedding_ready] +embeddings = get_embedding(text_content_list) +``` """ function extract_column_metadata(pg_conn_str::String)::DataFrame conn = LibPQ.Connection(pg_conn_str) @@ -457,6 +511,30 @@ julia> payloads = GeneralUtils.generate_embedding_payloads(df) Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text")) Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text")) ``` + +# Integration Guide +This function transforms column metadata into a format suitable for semantic search. +Use `text_content` field to generate embeddings, then compare against user question embeddings: + +```julia +# 1. Extract metadata +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) + +# 2. Generate payloads +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) + +# 3. Extract text content for embedding (only this field is used for similarity) +text_content_list = [i["text_content"] for i in embedding_ready] + +# 4. Generate embeddings for all columns +column_embeddings = get_embedding(text_content_list) + +# 5. Generate embedding for user question +question_embedding = get_embedding([user_question]) + +# 6. Calculate cosine similarity to find most relevant columns/tables +# (See harvest_db_undirected_schema_graph for full integration example) +``` """ function generate_embedding_payloads(df::DataFrame)::Vector{Dict} payloads = Dict[] @@ -537,6 +615,50 @@ julia> vector_hits = ["users", "products"] julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) ["users", "orders", "products"] ``` + +# Integration Guide: Understanding Table Relationships +This function determines how tables from vector search results are connected in the database schema. +The returned list represents the optimal join order for constructing SQL queries. + +**How tables are linked:** +- **Explicit Foreign Keys**: Direct relationships defined by `FOREIGN KEY` constraints in PostgreSQL +- **Implicit Relationships**: Tables sharing similar column naming patterns (e.g., `user_id` in both `users` and `orders` tables) + +**Example output interpretation:** +```julia +# Input: Top 20 tables from semantic search +top_tables = ["users", "products", "payments"] + +# Output: Tables in join order +table_relationship = ["users", "orders", "payments"] +# Interpretation: +# 1. Start with 'users' table +# 2. Join 'orders' via foreign key (likely users.id -> orders.user_id) +# 3. Join 'payments' via foreign key (likely orders.id -> payments.order_id) +``` + +**Full workflow example:** +```julia +# Step 1: Get column embeddings and find semantically related tables +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) +text_content = [i["text_content"] for i in embedding_ready] +table_embedding = get_embedding(text_content) + +# Step 2: Calculate similarity with user question +question_embedding = Float64.(get_embedding([question])["data"][1]["embedding"]) +similarities = [1 - Distances.cosine_dist(Float64.(i["embedding"]), question_embedding) + for i in table_embedding["data"]] + +# Step 3: Extract top related tables +top_tables = unique(metadata_df[sortperm(similarities)[1:20], :table_name]) + +# Step 4: Build schema graph and resolve relationships +g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str) +related_tables = GeneralUtils.resolve_semantic_cluster(top_tables, g, table_to_id, id_to_table) + +# related_tables now contains tables in optimal join order for SQL query construction +``` """ function resolve_semantic_cluster( vector_hits::Vector{String}, From 6fb2d5f82b6f280af346ae1bccdb788b43035731 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 15 Jul 2026 08:29:21 +0700 Subject: [PATCH 09/10] update --- src/llmUtil.jl | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/llmUtil.jl b/src/llmUtil.jl index f2c603b..888f7ae 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -767,6 +767,54 @@ julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name") ["Apple", "Banana", "Orange", "Mango"] ``` + +# Integration Guide: Finding Closest Entity Match in Database +When users type queries, they often make typos or use abbreviations. This function finds +the closest matching entity from a database column, handling common input errors. + +**Example:** +```julia +# Database contains: ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"] +# User types: "HandOld" (compressed words, missing space) + +catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "venues", "name") +closest = GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.9) +# Returns: "Hand Old Bar & Grill" (closest match) +``` + +**Step 1: Extract all valid values from a column** +```julia +# Get all wine names from the database +wine_names = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name") +# Returns: ["Château Margaux", "Château Lafite", "Domaine Leroy", ...] +``` + +**Step 2: Find closest match for user input** +```julia +# User types a wine name with a typo +catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name") +closest_wine = GeneralUtils.resolve_entity("Chateu Margaux", catalog; threshold=0.9) +# Returns: "Château Margaux" (closest match) +``` + +**Step 3: Resolve all fields in a response** +```julia +# Agent produces a response with potentially misspelled values +responsedict = Dict("wine_name" => "Chateu Margaux", "region" => "Bord") + +# For each field, find the closest match in the database +for (k, v) in responsedict + catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", k) + resolved = GeneralUtils.resolve_entity(v, catalog; threshold=0.9) + responsedict[k] = resolved # Replace messy input with database entity +end +``` + +**Threshold guidance:** +- `threshold=0.5` - Lenient, may return false positives +- `threshold=0.7` - Moderate balance +- `threshold=0.9` - Strict, only high-confidence matches +- Returns `""` when no match meets threshold """ function harvest_entity_catalog(pg_conn_str::String, table::String, column::String)::Vector{String} conn = LibPQ.Connection(pg_conn_str) @@ -832,6 +880,54 @@ julia> GeneralUtils.resolve_entity("Wine Cellar", catalog; threshold=0.5) julia> GeneralUtils.resolve_entity("Unknown Place", catalog; threshold=0.5) "" ``` + +# Integration Guide: Finding Closest Entity Match in Database +When users type queries, they often make typos or use abbreviations. This function finds +the closest matching entity from a database column, handling common input errors. + +**Example:** +```julia +# Database contains: ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"] +# User types: "HandOld" (compressed words, missing space) + +catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "venues", "name") +closest = GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.9) +# Returns: "Hand Old Bar & Grill" (closest match) +``` + +**Step 1: Extract all valid values from a column** +```julia +# Get all wine names from the database +wine_names = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name") +# Returns: ["Château Margaux", "Château Lafite", "Domaine Leroy", ...] +``` + +**Step 2: Find closest match for user input** +```julia +# User types a wine name with a typo +catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name") +closest_wine = GeneralUtils.resolve_entity("Chateu Margaux", catalog; threshold=0.9) +# Returns: "Château Margaux" (closest match) +``` + +**Step 3: Resolve all fields in a response** +```julia +# Agent produces a response with potentially misspelled values +responsedict = Dict("wine_name" => "Chateu Margaux", "region" => "Bord") + +# For each field, find the closest match in the database +for (k, v) in responsedict + catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", k) + resolved = GeneralUtils.resolve_entity(v, catalog; threshold=0.9) + responsedict[k] = resolved # Replace messy input with database entity +end +``` + +**Threshold guidance:** +- `threshold=0.5` - Lenient, may return false positives +- `threshold=0.7` - Moderate balance +- `threshold=0.9` - Strict, only high-confidence matches +- Returns `""` when no match meets threshold """ function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String best_match = "" From b09efc90682acc338bff34d6325fe77e59e54800 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 15 Jul 2026 11:16:30 +0700 Subject: [PATCH 10/10] update --- etc.jl | 293 ------------------------------------------------- src/llmUtil.jl | 52 ++++++++- 2 files changed, 51 insertions(+), 294 deletions(-) diff --git a/etc.jl b/etc.jl index 570b067..e69de29 100644 --- a/etc.jl +++ b/etc.jl @@ -1,293 +0,0 @@ - - - -using LibPQ, JSON, Graphs, DataFrames -config = JSON.parsefile("./appconfig.json") -host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':') -port = parse(Int, _port) -dbname = "winedb" -user = config["externalservice"]["sommpanion_db"]["user"] -password = config["externalservice"]["sommpanion_db"]["password"] -pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password" -db_connection = LibPQ.Connection(pg_conn_str) - - - - - -# ---------------------------------------------- 100 --------------------------------------------- # - - -using LibPQ -using DataFrames - -""" - extract_column_metadata(pg_conn_str::String) -> DataFrame - -Queries PostgreSQL system catalogs to extract a rich semantic text map of every -column in the database. Returns a DataFrame designed for vector embedding generation. -""" -function extract_column_metadata(pg_conn_str::String) - conn = LibPQ.Connection(pg_conn_str) - - # This direct SQL query pulls the column specifications along with column-level descriptions - query = """ - SELECT - c.relname AS table_name, - a.attname AS column_name, - format_type(a.atttypid, a.atttypmod) AS data_type, - COALESCE(d.description, '') AS column_description, - CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key, - CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key - FROM pg_attribute a - JOIN pg_class c ON c.oid = a.attrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - -- Join to fetch column comments/descriptions - LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum - -- Check if column is part of a Primary Key - LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid - AND pk.contype = 'p' - AND a.attnum = ANY(pk.conkey) - -- Check if column is part of a Foreign Key - LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid - AND fk.contype = 'f' - AND a.attnum = ANY(fk.conkey) - WHERE - n.nspname = 'public' -- Only user schemas - AND c.relkind = 'r' -- Only standard tables - AND a.attnum > 0 -- Skip system hidden columns - AND NOT a.attisdropped; -- Skip dropped columns - """ - - try - # Execute and format into a clean DataFrame - result = execute(conn, query) - df = DataFrame(result) - - # Create a unique document ID for each vector row - df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)] - - return df - finally - close(conn) - end -end - - - -""" - generate_embedding_payloads(df::DataFrame) -> Vector{Dict} - -Transforms the metadata DataFrame into structured text strings optimal for -vector space mapping. -""" -function generate_embedding_payloads(df::DataFrame) - payloads = Dict[] - - for row in eachrow(df) - # 1. Build a rich text description summarizing the column's role - text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)" - - if row.is_primary_key - text_payload *= " [PRIMARY KEY]" - end - if row.is_foreign_key - text_payload *= " [FOREIGN KEY RELATIONAL LINK]" - end - - # Append business descriptions if they exist in the database comments - if !isempty(strip(row.column_description)) - text_payload *= " | Description: $(row.column_description)" - else - text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture." - end - - # 2. Package everything neatly to be passed to your vector store client - push!(payloads, Dict( - "id" => row.vector_id, - "text_content" => text_payload, - "metadata" => Dict( - "table" => row.table_name, - "column" => row.column_name, - "type" => row.data_type - ) - )) - end - - return payloads -end - - -""" - resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String} - -Takes a scattered array of semantically matched tables from Stage 1, navigates -the undirected network structure, and isolates the minimum interconnected subgraph -required to weave ALL hits into a single valid SQL query. -""" -function resolve_semantic_cluster( - vector_hits::Vector{String}, - g::SimpleGraph, - table_to_id::Dict{String, Int}, - id_to_table::Dict{Int, String} -) - # Filter out hits that don't exist in our actual database graph mapping - valid_node_ids = Int[] - for hit in vector_hits - if haskey(table_to_id, hit) - push!(valid_node_ids, table_to_id[hit]) - else - @warn "Vector hit '$hit' does not map to an existing database table." - end - end - - unique!(valid_node_ids) - - # Edge Case Handlers - if isempty(valid_node_ids) - return String[] - elseif length(valid_node_ids) == 1 - return [id_to_table[valid_node_ids[1]]] - end - - # The Isolated Subgraph Set to build our final context - schema_subgraph_nodes = Set{Int}() - - # Phase A: Select an initial anchor component. We use the highest-ranked vector hit. - anchor_node = valid_node_ids[1] - push!(schema_subgraph_nodes, anchor_node) - - # Phase B: Sequentially route paths to all other semantic coordinates - for target_node in valid_node_ids[2:end] - # Skip if an earlier loop trajectory already naturally absorbed this table - if target_node in schema_subgraph_nodes - continue - end - - # Calculate the shortest path tree from the CURRENT state of our subgraph - # We find the shortest path from the target back to ANY node currently in our tree - shortest_paths = dijkstra_shortest_paths(g, target_node) - - # Find which node currently in our subgraph is closest to the target node - closest_subgraph_node = 0 - min_distance = Inf - - for subgraph_node in schema_subgraph_nodes - dist = shortest_paths.dists[subgraph_node] - if dist < min_distance - min_distance = dist - closest_subgraph_node = subgraph_node - end - end - - # Reconstruct the path from the target node to the closest point on our existing tree - if closest_subgraph_node != 0 - curr = closest_subgraph_node - while curr != 0 - push!(schema_subgraph_nodes, curr) - curr = shortest_paths.parents[curr] - if curr == target_node - push!(schema_subgraph_nodes, target_node) - break - end - end - end - end - - # Map the unique structural nodes back to clean table names - return [id_to_table[node_id] for node_id in schema_subgraph_nodes] -end - -function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String}) - documents_dict = Dict("documents" => text) - payloads = [("documents", documents_dict, "dictionary")] - _, msg_envelope_json_str = msghandler.smartpack( - config["externalservice"]["servicesloadbalancer"]["nats"], - payloads; - msg_purpose="embedding", - broker_url=config["nats_server_info"]["url"], - fileserver_url=config["externalservice"]["fileserver"]["url"]) - - reply = NATS.request(nats_conn, - config["externalservice"]["servicesloadbalancer"]["nats"], - msg_envelope_json_str, timeout=120) - incoming_env_json_str = String(reply.payload) - incoming_env = msghandler.smartunpack(incoming_env_json_str) - embedding_response = incoming_env["payloads"][1][2] - - return embedding_response -end - -nats_conn = NATS.connect(config["nats_server_info"]["url"]) - -# Run the extractor -metadata_df = extract_column_metadata(pg_conn_str) -embedding_ready = generate_embedding_payloads(metadata_df) - -println(embedding_ready[1]["text_content"]) -# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items." - -# use only text content -embedding_ready_2 = [i["text_content"] for i in embedding_ready] -table_embedding = get_embedding(nats_conn, embedding_ready_2) - -user_question = - """ - Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b} - """ -user_question_embedding = get_embedding(nats_conn, [user_question]) - - -using Distances -# similarity = 1 - Distances.cosine_dist(Float64.(table_embedding["data"][1]["embedding"]), -# Float64.(user_question_embedding["data"][1]["embedding"]) -# ) -user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"]) -user_question_similarity = [] -for i in table_embedding["data"] - i_data = i["embedding"] - i_float = Float64.(i_data) - r = 1 - Distances.cosine_dist(i_float, user_question_embedding) - push!(user_question_similarity, r) -end - -new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) -sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min - -# top 20 of sorted_df get this tables -vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"] - -g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str) - -# tables that I should put schema in LLM context -optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) - - - -function find_related_tables_for_user_question(question::String) - metadata_df = extract_column_metadata(pg_conn_str) - embedding_ready = generate_embedding_payloads(metadata_df) - - # use only text content - embedding_ready_2 = [i["text_content"] for i in embedding_ready] - table_embedding = get_embedding(nats_conn, embedding_ready_2) - user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"]) - user_question_similarity = [] - for i in table_embedding["data"] - i_data = i["embedding"] - i_float = Float64.(i_data) - r = 1 - Distances.cosine_dist(i_float, user_question_embedding) - push!(user_question_similarity, r) - end - - new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) - sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min - - #WORKING extract top 20 rows of sorted_df to get tables that related to user question - vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"] - - g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str) - - # tables that I should put schema in LLM context - return optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) -end diff --git a/src/llmUtil.jl b/src/llmUtil.jl index 888f7ae..1c5e9d8 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -2,7 +2,8 @@ module llmUtil export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response, extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster, - harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph + harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph, + get_db_table_schema using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ using ..util @@ -964,6 +965,55 @@ function resolve_entity(messy_input::String, catalog::Vector{String}; threshold= end +function get_db_table_schema(pg_conn_str::String, table_name::String)::DataFrame + conn = LibPQ.Connection(pg_conn_str) + return get_db_table_schema(conn, table_name) +end + +function get_db_table_schema(conn::LibPQ.Connection, table_name::String)::DataFrame + # This direct SQL query pulls the column specifications along with column-level descriptions + query = """ + SELECT + a.attname AS column_name, + format_type(a.atttypid, a.atttypmod) AS data_type, + COALESCE(d.description, '') AS column_comment, + CASE + WHEN p.contype = 'p' THEN 'PRIMARY KEY' + WHEN p.contype = 'u' THEN 'UNIQUE' + WHEN p.contype = 'f' THEN 'FOREIGN KEY' + WHEN p.contype = 'c' THEN 'CHECK' + ELSE '' + END AS constraint_type, + COALESCE(p.conname, '') AS constraint_name + FROM + pg_catalog.pg_attribute a + JOIN + pg_catalog.pg_class c ON a.attrelid = c.oid + JOIN + pg_catalog.pg_namespace n ON c.relnamespace = n.oid + LEFT JOIN + pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum + LEFT JOIN + pg_catalog.pg_constraint p ON p.conrelid = c.oid AND a.attnum = ANY(p.conkey) + WHERE + c.relname = '$table_name' -- <-- Put your table name here + AND n.nspname = 'public' -- <-- Change schema if not 'public' + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY + a.attnum; + """ + + try + # Execute and format into a clean DataFrame + result = LibPQ.execute(conn, query) + df = DataFrame(result) + + return df + finally + close(conn) + end +end