update

Reviewed-on: #12

Project.toml

@@ -1,6 +1,6 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.4.10"
+version = "0.5.11"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
 
 [deps]

etc.jl

@@ -1,293 +0,0 @@
-
-
-
-using LibPQ, JSON, Graphs, DataFrames
-config = JSON.parsefile("./appconfig.json")
-host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
-port = parse(Int, _port)
-dbname = "winedb"
-user = config["externalservice"]["sommpanion_db"]["user"]
-password = config["externalservice"]["sommpanion_db"]["password"]
-pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password"
-db_connection = LibPQ.Connection(pg_conn_str)
-
-
-
-
-
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-
-using LibPQ
-using DataFrames
-
-"""
-    extract_column_metadata(pg_conn_str::String) -> DataFrame
-
-Queries PostgreSQL system catalogs to extract a rich semantic text map of every 
-column in the database. Returns a DataFrame designed for vector embedding generation.
-"""
-function extract_column_metadata(pg_conn_str::String)
-    conn = LibPQ.Connection(pg_conn_str)
-    
-    # This direct SQL query pulls the column specifications along with column-level descriptions
-    query = """
-        SELECT 
-            c.relname AS table_name,
-            a.attname AS column_name,
-            format_type(a.atttypid, a.atttypmod) AS data_type,
-            COALESCE(d.description, '') AS column_description,
-            CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
-            CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key
-        FROM pg_attribute a
-        JOIN pg_class c ON c.oid = a.attrelid
-        JOIN pg_namespace n ON n.oid = c.relnamespace
-        -- Join to fetch column comments/descriptions
-        LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
-        -- Check if column is part of a Primary Key
-        LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid 
-            AND pk.contype = 'p' 
-            AND a.attnum = ANY(pk.conkey)
-        -- Check if column is part of a Foreign Key
-        LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid 
-            AND fk.contype = 'f' 
-            AND a.attnum = ANY(fk.conkey)
-        WHERE 
-            n.nspname = 'public'          -- Only user schemas
-            AND c.relkind = 'r'            -- Only standard tables
-            AND a.attnum > 0               -- Skip system hidden columns
-            AND NOT a.attisdropped;        -- Skip dropped columns
-    """
-    
-    try
-        # Execute and format into a clean DataFrame
-        result = execute(conn, query)
-        df = DataFrame(result)
-        
-        # Create a unique document ID for each vector row
-        df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
-        
-        return df
-    finally
-        close(conn)
-    end
-end
-
-
-
-"""
-    generate_embedding_payloads(df::DataFrame) -> Vector{Dict}
-
-Transforms the metadata DataFrame into structured text strings optimal for 
-vector space mapping.
-"""
-function generate_embedding_payloads(df::DataFrame)
-    payloads = Dict[]
-    
-    for row in eachrow(df)
-        # 1. Build a rich text description summarizing the column's role
-        text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
-        
-        if row.is_primary_key
-            text_payload *= " [PRIMARY KEY]"
-        end
-        if row.is_foreign_key
-            text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
-        end
-        
-        # Append business descriptions if they exist in the database comments
-        if !isempty(strip(row.column_description))
-            text_payload *= " | Description: $(row.column_description)"
-        else
-            text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
-        end
-        
-        # 2. Package everything neatly to be passed to your vector store client
-        push!(payloads, Dict(
-            "id" => row.vector_id,
-            "text_content" => text_payload,
-            "metadata" => Dict(
-                "table" => row.table_name,
-                "column" => row.column_name,
-                "type" => row.data_type
-            )
-        ))
-    end
-    
-    return payloads
-end
-
-
-"""
-    resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String}
-
-Takes a scattered array of semantically matched tables from Stage 1, navigates 
-the undirected network structure, and isolates the minimum interconnected subgraph 
-required to weave ALL hits into a single valid SQL query.
-"""
-function resolve_semantic_cluster(
-    vector_hits::Vector{String}, 
-    g::SimpleGraph, 
-    table_to_id::Dict{String, Int}, 
-    id_to_table::Dict{Int, String}
-)
-    # Filter out hits that don't exist in our actual database graph mapping
-    valid_node_ids = Int[]
-    for hit in vector_hits
-        if haskey(table_to_id, hit)
-            push!(valid_node_ids, table_to_id[hit])
-        else
-            @warn "Vector hit '$hit' does not map to an existing database table."
-        end
-    end
-    
-    unique!(valid_node_ids)
-    
-    # Edge Case Handlers
-    if isempty(valid_node_ids)
-        return String[]
-    elseif length(valid_node_ids) == 1
-        return [id_to_table[valid_node_ids[1]]]
-    end
-
-    # The Isolated Subgraph Set to build our final context
-    schema_subgraph_nodes = Set{Int}()
-    
-    # Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
-    anchor_node = valid_node_ids[1]
-    push!(schema_subgraph_nodes, anchor_node)
-    
-    # Phase B: Sequentially route paths to all other semantic coordinates
-    for target_node in valid_node_ids[2:end]
-        # Skip if an earlier loop trajectory already naturally absorbed this table
-        if target_node in schema_subgraph_nodes
-            continue
-        end
-        
-        # Calculate the shortest path tree from the CURRENT state of our subgraph
-        # We find the shortest path from the target back to ANY node currently in our tree
-        shortest_paths = dijkstra_shortest_paths(g, target_node)
-        
-        # Find which node currently in our subgraph is closest to the target node
-        closest_subgraph_node = 0
-        min_distance = Inf
-        
-        for subgraph_node in schema_subgraph_nodes
-            dist = shortest_paths.dists[subgraph_node]
-            if dist < min_distance
-                min_distance = dist
-                closest_subgraph_node = subgraph_node
-            end
-        end
-        
-        # Reconstruct the path from the target node to the closest point on our existing tree
-        if closest_subgraph_node != 0
-            curr = closest_subgraph_node
-            while curr != 0
-                push!(schema_subgraph_nodes, curr)
-                curr = shortest_paths.parents[curr]
-                if curr == target_node
-                    push!(schema_subgraph_nodes, target_node)
-                    break
-                end
-            end
-        end
-    end
-    
-    # Map the unique structural nodes back to clean table names
-    return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
-end
-
-function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String})
-  documents_dict = Dict("documents" => text)
-  payloads = [("documents", documents_dict, "dictionary")]
-  _, msg_envelope_json_str = msghandler.smartpack(
-    config["externalservice"]["servicesloadbalancer"]["nats"], 
-    payloads;   
-    msg_purpose="embedding",  
-    broker_url=config["nats_server_info"]["url"],
-    fileserver_url=config["externalservice"]["fileserver"]["url"])
-
-  reply = NATS.request(nats_conn, 
-                      config["externalservice"]["servicesloadbalancer"]["nats"], 
-                      msg_envelope_json_str, timeout=120)
-  incoming_env_json_str = String(reply.payload)
-  incoming_env = msghandler.smartunpack(incoming_env_json_str)
-  embedding_response = incoming_env["payloads"][1][2]
-
-  return embedding_response
-end
-
-nats_conn  = NATS.connect(config["nats_server_info"]["url"])
-
-# Run the extractor
-metadata_df = extract_column_metadata(pg_conn_str)
-embedding_ready = generate_embedding_payloads(metadata_df)
-
-println(embedding_ready[1]["text_content"])
-# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items."
-
-# use only text content
-embedding_ready_2 = [i["text_content"] for i in embedding_ready]
-table_embedding = get_embedding(nats_conn, embedding_ready_2)
-
-user_question = 
-  """
-  Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b}
-  """
-user_question_embedding = get_embedding(nats_conn, [user_question])
-
-
-using Distances
-# similarity = 1 - Distances.cosine_dist(Float64.(table_embedding["data"][1]["embedding"]), 
-#                             Float64.(user_question_embedding["data"][1]["embedding"])
-#                             )
-user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
-user_question_similarity = []
-for i in table_embedding["data"]
-  i_data = i["embedding"]
-  i_float = Float64.(i_data)
-  r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
-  push!(user_question_similarity, r)
-end
-
-new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
-sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
-
-# top 20 of sorted_df get this tables
-vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
-
-g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
-
-# tables that I should put schema in LLM context
-optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
-
-
-
-function find_related_tables_for_user_question(question::String)
-  metadata_df = extract_column_metadata(pg_conn_str)
-  embedding_ready = generate_embedding_payloads(metadata_df)
-
-  # use only text content
-  embedding_ready_2 = [i["text_content"] for i in embedding_ready]
-  table_embedding = get_embedding(nats_conn, embedding_ready_2)
-  user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
-  user_question_similarity = []
-  for i in table_embedding["data"]
-    i_data = i["embedding"]
-    i_float = Float64.(i_data)
-    r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
-    push!(user_question_similarity, r)
-  end
-
-  new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
-  sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
-
-  #WORKING extract top 20 rows of sorted_df to get tables that related to user question
-  vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
-
-  g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
-
-  # tables that I should put schema in LLM context
-  return optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
-end

src/dbUtil.jl

@@ -1,13 +1,107 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
 using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
-      SHA
+      SHA, NATS, LibPQ
 using ..util
 #[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
+""" Execute SQL against a PostgreSQL database using LibPQ connection string.
+
+# Arguments
+- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
+- `sql::AbstractString`: SQL query to execute
+
+# Returns
+- `LibPQ.Result` on success, `nothing` on failure
+
+# Example
+```julia
+pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
+sql = "SELECT * FROM wine;"
+result = execute_postgres_sql(pg_conn_str, sql)
+```
+"""
+function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
+  db_connection = LibPQ.Connection(pg_conn_str)
+  result = nothing
+  try
+    result = LibPQ.execute(db_connection, sql)  
+  catch e
+    @error e
+    LibPQ.close(db_connection)
+  end
+  
+  LibPQ.close(db_connection)
+  return result
+end
+
+
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
+
+# Arguments
+- `text::AbstractString`: The input text to find similar records for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
+
+# Returns
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
+
+# Example
+```julia
+# Assume you have embedding and SQL execution functions
+text = "a rich structured red wine"
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
+```
+"""
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
+  executesql::Function, get_embedding::Function; 
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
+end
+
 
 """
 dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format

src/llmUtil.jl

@@ -2,7 +2,8 @@ module llmUtil
 
 export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
         extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster,
-        harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph
+        harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph,
+        get_db_table_schema, get_db_table_schema_simple
 
 using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ
 using ..util
@@ -252,6 +253,45 @@ julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_g
 julia> vertices(g)
 10
 ```
+
+# Integration Guide: Finding Related Tables for User Questions
+This function is designed to work together with `extract_column_metadata` and `resolve_semantic_cluster` to answer user questions by identifying related tables:
+
+```julia
+# Step 1: Extract column metadata and generate embeddings for semantic search
+pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+
+# Use only text content for embedding
+embedding_ready_2 = [i["text_content"] for i in embedding_ready]
+table_embedding = get_embedding(embedding_ready_2)
+
+# Get embedding for user question
+_user_question_embedding = get_embedding([question])
+user_question_embedding = Float64.(_user_question_embedding["data"][1]["embedding"])
+
+# Calculate similarity between question and all columns
+user_question_similarity = []
+for i in table_embedding["data"]
+  i_data = i["embedding"]
+  i_float = Float64.(i_data)
+  r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
+  push!(user_question_similarity, r)
+end
+
+# Step 2: Find top related tables
+new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
+sorted_df = sort(new_df, :user_question_similarity, rev=true)
+_top_20_tables = unique(sorted_df[1:20, :table_name])
+top_20_tables = [i for i in _top_20_tables]
+
+# Step 3: Build schema graph and resolve table relationships
+g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str)
+table_relationship = GeneralUtils.resolve_semantic_cluster(top_20_tables, g, table_to_id, id_to_table)
+
+# table_relationship now contains tables in the order they should be joined
+```
 """
 function harvest_db_undirected_schema_graph(pg_conn_str)   
   conn = LibPQ.Connection(pg_conn_str) 
@@ -371,6 +411,21 @@ products    id           integer    Product ID          true            false
 products    price        numeric    Product price       false           false         col_products_price
 products    user_id      integer    Reference to user   false           true          col_products_user_id
 ```
+
+# Integration Guide
+This function is the first step in the semantic table discovery pipeline. Use it with `generate_embedding_payloads` to prepare data for vector similarity search:
+
+```julia
+# Step 1: Extract metadata
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+
+# Step 2: Generate embedding payloads for semantic search
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+
+# Use text_content field for embedding
+text_content_list = [i["text_content"] for i in embedding_ready]
+embeddings = get_embedding(text_content_list)
+```
 """
 function extract_column_metadata(pg_conn_str::String)::DataFrame
   conn = LibPQ.Connection(pg_conn_str)
@@ -457,6 +512,30 @@ julia> payloads = GeneralUtils.generate_embedding_payloads(df)
  Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text"))
  Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text"))
 ```
+
+# Integration Guide
+This function transforms column metadata into a format suitable for semantic search.
+Use `text_content` field to generate embeddings, then compare against user question embeddings:
+
+```julia
+# 1. Extract metadata
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+
+# 2. Generate payloads
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+
+# 3. Extract text content for embedding (only this field is used for similarity)
+text_content_list = [i["text_content"] for i in embedding_ready]
+
+# 4. Generate embeddings for all columns
+column_embeddings = get_embedding(text_content_list)
+
+# 5. Generate embedding for user question
+question_embedding = get_embedding([user_question])
+
+# 6. Calculate cosine similarity to find most relevant columns/tables
+# (See harvest_db_undirected_schema_graph for full integration example)
+```
 """
 function generate_embedding_payloads(df::DataFrame)::Vector{Dict}
     payloads = Dict[]
@@ -537,6 +616,50 @@ julia> vector_hits = ["users", "products"]
 julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
 ["users", "orders", "products"]
 ```
+
+# Integration Guide: Understanding Table Relationships
+This function determines how tables from vector search results are connected in the database schema.
+The returned list represents the optimal join order for constructing SQL queries.
+
+**How tables are linked:**
+- **Explicit Foreign Keys**: Direct relationships defined by `FOREIGN KEY` constraints in PostgreSQL
+- **Implicit Relationships**: Tables sharing similar column naming patterns (e.g., `user_id` in both `users` and `orders` tables)
+
+**Example output interpretation:**
+```julia
+# Input: Top 20 tables from semantic search
+top_tables = ["users", "products", "payments"]
+
+# Output: Tables in join order
+table_relationship = ["users", "orders", "payments"]
+# Interpretation: 
+#   1. Start with 'users' table
+#   2. Join 'orders' via foreign key (likely users.id -> orders.user_id)
+#   3. Join 'payments' via foreign key (likely orders.id -> payments.order_id)
+```
+
+**Full workflow example:**
+```julia
+# Step 1: Get column embeddings and find semantically related tables
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+text_content = [i["text_content"] for i in embedding_ready]
+table_embedding = get_embedding(text_content)
+
+# Step 2: Calculate similarity with user question
+question_embedding = Float64.(get_embedding([question])["data"][1]["embedding"])
+similarities = [1 - Distances.cosine_dist(Float64.(i["embedding"]), question_embedding) 
+                for i in table_embedding["data"]]
+
+# Step 3: Extract top related tables
+top_tables = unique(metadata_df[sortperm(similarities)[1:20], :table_name])
+
+# Step 4: Build schema graph and resolve relationships
+g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str)
+related_tables = GeneralUtils.resolve_semantic_cluster(top_tables, g, table_to_id, id_to_table)
+
+# related_tables now contains tables in optimal join order for SQL query construction
+```
 """
 function resolve_semantic_cluster(
     vector_hits::Vector{String}, 
@@ -645,6 +768,54 @@ julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret
 julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name")
 ["Apple", "Banana", "Orange", "Mango"]
 ```
+
+# Integration Guide: Finding Closest Entity Match in Database
+When users type queries, they often make typos or use abbreviations. This function finds
+the closest matching entity from a database column, handling common input errors.
+
+**Example:**
+```julia
+# Database contains: ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"]
+# User types: "HandOld" (compressed words, missing space)
+
+catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "venues", "name")
+closest = GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.9)
+# Returns: "Hand Old Bar & Grill" (closest match)
+```
+
+**Step 1: Extract all valid values from a column**
+```julia
+# Get all wine names from the database
+wine_names = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name")
+# Returns: ["Château Margaux", "Château Lafite", "Domaine Leroy", ...]
+```
+
+**Step 2: Find closest match for user input**
+```julia
+# User types a wine name with a typo
+catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name")
+closest_wine = GeneralUtils.resolve_entity("Chateu Margaux", catalog; threshold=0.9)
+# Returns: "Château Margaux" (closest match)
+```
+
+**Step 3: Resolve all fields in a response**
+```julia
+# Agent produces a response with potentially misspelled values
+responsedict = Dict("wine_name" => "Chateu Margaux", "region" => "Bord")
+
+# For each field, find the closest match in the database
+for (k, v) in responsedict
+  catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", k)
+  resolved = GeneralUtils.resolve_entity(v, catalog; threshold=0.9)
+  responsedict[k] = resolved  # Replace messy input with database entity
+end
+```
+
+**Threshold guidance:**
+- `threshold=0.5` - Lenient, may return false positives
+- `threshold=0.7` - Moderate balance
+- `threshold=0.9` - Strict, only high-confidence matches
+- Returns `""` when no match meets threshold
 """
 function harvest_entity_catalog(pg_conn_str::String, table::String, column::String)::Vector{String}
   conn = LibPQ.Connection(pg_conn_str)
@@ -710,6 +881,54 @@ julia> GeneralUtils.resolve_entity("Wine Cellar", catalog; threshold=0.5)
 julia> GeneralUtils.resolve_entity("Unknown Place", catalog; threshold=0.5)
 ""
 ```
+
+# Integration Guide: Finding Closest Entity Match in Database
+When users type queries, they often make typos or use abbreviations. This function finds
+the closest matching entity from a database column, handling common input errors.
+
+**Example:**
+```julia
+# Database contains: ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"]
+# User types: "HandOld" (compressed words, missing space)
+
+catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "venues", "name")
+closest = GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.9)
+# Returns: "Hand Old Bar & Grill" (closest match)
+```
+
+**Step 1: Extract all valid values from a column**
+```julia
+# Get all wine names from the database
+wine_names = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name")
+# Returns: ["Château Margaux", "Château Lafite", "Domaine Leroy", ...]
+```
+
+**Step 2: Find closest match for user input**
+```julia
+# User types a wine name with a typo
+catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", "wine_name")
+closest_wine = GeneralUtils.resolve_entity("Chateu Margaux", catalog; threshold=0.9)
+# Returns: "Château Margaux" (closest match)
+```
+
+**Step 3: Resolve all fields in a response**
+```julia
+# Agent produces a response with potentially misspelled values
+responsedict = Dict("wine_name" => "Chateu Margaux", "region" => "Bord")
+
+# For each field, find the closest match in the database
+for (k, v) in responsedict
+  catalog = GeneralUtils.harvest_entity_catalog(pg_conn_str, "wine", k)
+  resolved = GeneralUtils.resolve_entity(v, catalog; threshold=0.9)
+  responsedict[k] = resolved  # Replace messy input with database entity
+end
+```
+
+**Threshold guidance:**
+- `threshold=0.5` - Lenient, may return false positives
+- `threshold=0.7` - Moderate balance
+- `threshold=0.9` - Strict, only high-confidence matches
+- Returns `""` when no match meets threshold
 """
 function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String
     best_match = ""
@@ -746,9 +965,166 @@ function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=
 end
 
 
+function get_db_table_schema(pg_conn_str::String, table_name::String)::DataFrame
+  conn = LibPQ.Connection(pg_conn_str)
+  return get_db_table_schema(conn, table_name)
+end
+
+function get_db_table_schema(conn::LibPQ.Connection, table_name::String)::DataFrame
+  # This direct SQL query pulls the column specifications along with column-level descriptions
+  query = """
+        SELECT 
+            a.attname AS column_name,
+            format_type(a.atttypid, a.atttypmod) AS data_type,
+            COALESCE(d.description, '') AS column_comment,
+            CASE 
+                WHEN p.contype = 'p' THEN 'PRIMARY KEY'
+                WHEN p.contype = 'u' THEN 'UNIQUE'
+                WHEN p.contype = 'f' THEN 'FOREIGN KEY'
+                WHEN p.contype = 'c' THEN 'CHECK'
+                ELSE ''
+            END AS constraint_type,
+            COALESCE(p.conname, '') AS constraint_name
+        FROM 
+            pg_catalog.pg_attribute a
+        JOIN 
+            pg_catalog.pg_class c ON a.attrelid = c.oid
+        JOIN 
+            pg_catalog.pg_namespace n ON c.relnamespace = n.oid
+        LEFT JOIN 
+            pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
+        LEFT JOIN 
+            pg_catalog.pg_constraint p ON p.conrelid = c.oid AND a.attnum = ANY(p.conkey)
+        WHERE 
+            c.relname = '$table_name' -- <-- Put your table name here
+            AND n.nspname = 'public'      -- <-- Change schema if not 'public'
+            AND a.attnum > 0 
+            AND NOT a.attisdropped
+        ORDER BY 
+            a.attnum;
+    """
+  
+  try
+      # Execute and format into a clean DataFrame
+      result = LibPQ.execute(conn, query)
+      df = DataFrame(result)
+      
+      return df
+  finally
+      close(conn)
+  end
+end
 
 
+""" Generate simplified DDL statement for a PostgreSQL table.
 
+Extracts table structure from PostgreSQL and returns a clean CREATE TABLE statement
+without NULL/NOT NULL constraints, useful for schema documentation or migration purposes.
+
+# Arguments
+- `conn::LibPQ.Connection`
+  A PostgreSQL connection object created via `LibPQ.Connection()`.
+- `table_name::String`
+  The name of the table to extract.
+- `schema_name::String` (default: `"public"`)
+  The schema containing the table.
+
+# Return
+- `String`
+  A CREATE TABLE DDL statement containing:
+  - Column definitions with names, types, and DEFAULT values
+  - Table-level constraints (PRIMARY KEY, FOREIGN KEY, UNIQUE, CHECK)
+  - Excludes NULL/NOT NULL specifications for cleaner output
+
+# Details
+The function:
+1. Queries PostgreSQL system catalogs (`pg_attribute`, `pg_class`, `pg_namespace`)
+2. Extracts column names, data types, and default values
+3. Captures table-level constraints via `pg_constraint`
+4. Omits NULL/NOT NULL checks to produce a simplified schema definition
+5. Returns properly formatted DDL with quoted identifiers
+
+# Example
+```julia
+julia> using GeneralUtils, LibPQ
+julia> conn = LibPQ.Connection("host=localhost port=5432 dbname=winedb user=admin password=secret")
+julia> ddl = GeneralUtils.get_db_table_schema_simple(conn, "wine")
+"CREATE TABLE \"public\".\"wine\" (
+    \"id\" integer DEFAULT nextval('wine_id_seq'::regclass),
+    \"wine_name\" text,
+    \"year\" integer,
+    \"price\" numeric
+);"
+```
+"""
+function get_db_table_schema_simple(pg_conn_str::String, table_name::String; 
+  schema_name::String="public")::String
+  conn = LibPQ.Connection(pg_conn_str)
+  return get_db_table_schema_simple(conn, table_name; schema_name=schema_name)
+end
+
+function get_db_table_schema_simple(conn, table_name::String; schema_name::String="public")::String
+    # 1. SQL query tailored to omit nullability checks
+    sql = """
+    SELECT 
+        a.attname AS column_name,
+        format_type(a.atttypid, a.atttypmod) AS data_type,
+        pg_get_expr(def.adbin, def.adrelid) AS default_value,
+        COALESCE(
+            (SELECT pg_get_constraintdef(p.oid) 
+             FROM pg_catalog.pg_constraint p 
+             WHERE p.conrelid = c.oid AND a.attnum = ANY(p.conkey) 
+             LIMIT 1), ''
+        ) AS constraint_definition
+    FROM 
+        pg_catalog.pg_attribute a
+    JOIN 
+        pg_catalog.pg_class c ON a.attrelid = c.oid
+    JOIN 
+        pg_catalog.pg_namespace n ON c.relnamespace = n.oid
+    LEFT JOIN 
+        pg_catalog.pg_attrdef def ON def.adrelid = c.oid AND def.adnum = a.attnum
+    WHERE 
+        c.relname = \$1
+        AND n.nspname = \$2
+        AND a.attnum > 0 
+        AND NOT a.attisdropped
+    ORDER BY 
+        a.attnum;
+    """
+
+    result = execute(conn, sql, [table_name, schema_name])
+    
+    if length(result) == 0
+        error("Table '$schema_name.$table_name' not found.")
+    end
+
+    ddl_lines = String[]
+    constraints = String[]
+
+    for row in result
+        col_name   = row.column_name
+        data_type  = row.data_type
+        
+        # Handle the default value if it exists
+        default_val = ismissing(row.default_value) ? "" : " DEFAULT " * row.default_value
+        
+        # Build the column definition line (without NULL/NOT NULL)
+        col_def = "    \"$col_name\" $data_type$default_val"
+        push!(ddl_lines, col_def)
+
+        # Handle table-level constraints
+        con_def = ismissing(row.constraint_definition) ? "" : row.constraint_definition
+        if !isempty(con_def) && !(con_def in constraints)
+            push!(constraints, "    " * con_def)
+        end
+    end
+
+    all_definitions = vcat(ddl_lines, constraints)
+    body = join(all_definitions, ",\n")
+    
+    return "CREATE TABLE \"$schema_name\".\"$table_name\" (\n$body\n);"
+end