update

src/llmUtil.jl

@@ -1,9 +1,10 @@
 module llmUtil
 
-export formatLLMtext, extractthink,
-        checkAgentResponse_JSON, clean_json_response
+export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
+        extract_vector_metadata, generate_embedding_payloads, resolve_semantic_cluster,
+        harvest_entity_catalog, resolve_entity
 
-using UUIDs, JSON, Dates, DataFrames
+using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs
 using GeneralUtils
 
 # ---------------------------------------------- 100 --------------------------------------------- #
@@ -228,7 +229,7 @@ Returns a DataFrame designed for vector embedding generation.
 
 # Arguments
 - `pg_conn_str::String`
-  PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname")
+  PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
 
 # Return
 - `DataFrame`
@@ -244,7 +245,7 @@ Returns a DataFrame designed for vector embedding generation.
 # Example
 ```julia
 julia> using GeneralUtils
-julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
+julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
 julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
 DataFrame
 6 rows × 7 columns
@@ -333,7 +334,7 @@ The function constructs rich text payloads by:
 # Example
 ```julia
 julia> using GeneralUtils
-julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
+julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
 julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
 julia> payloads = GeneralUtils.generate_embedding_payloads(df)
 3-element Vector{Dict}:
@@ -496,10 +497,132 @@ function resolve_semantic_cluster(
 end
 
 
+""" Harvest entity catalog from database column.
+
+Extracts unique, non-null values from a specific column to build a local index for 
+semantic search or entity resolution.
+
+# Arguments
+- `conn_str::String`
+  PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
+- `table::String`
+  Table name to query
+- `column::String`
+  Column name containing entity values
+
+# Return
+- `Vector{String}`
+  A vector of unique, stripped strings from the specified column. Empty strings
+  are removed via `strip()`.
+
+# Details
+The function:
+1. Connects to PostgreSQL database
+2. Executes `SELECT DISTINCT column FROM table WHERE column IS NOT NULL`
+3. Converts result to DataFrame
+4. Strips whitespace from each value and converts to String
+5. Returns clean vector of unique entity values
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name")
+["Apple", "Banana", "Orange", "Mango"]
+```
+"""
+function harvest_entity_catalog(conn_str::String, table::String, column::String)::Vector{String}
+    conn = LibPQ.Connection(conn_str)
+    
+    # We only care about unique, non-null values to keep the index fast and dense
+    query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;"
+    
+    try
+        df = DataFrame(execute(conn, query))
+        # Return as a clean array of strings
+        return String.(strip.(df[:, 1]))
+    finally
+        close(conn)
+    end
+end
 
 
+""" Resolve entity name from messy input using fuzzy string matching.
 
+Matches user-provided text against a reference catalog using Jaro-Winkler similarity
+and returns the closest matching exact string from the database catalog.
 
+# Arguments
+- `messy_input::String`
+  The user input text that may contain typos, compressed words, or variations.
+- `catalog::Vector{String}`
+  A vector of valid, exact entity strings from the database.
+
+# Keyword Arguments
+- `threshold::Float64` (default: `0.5`)
+  Minimum similarity score (0.0 to 1.0) required to return a match. Lower values
+  allow more lenient matching; higher values require closer matches.
+
+# Return
+- `String`
+  The exact matching string from `catalog` if similarity score ≥ threshold,
+  otherwise an empty string `""`.
+
+# Details
+The function:
+1. Normalizes input to lowercase and strips whitespace
+2. Computes Jaro-Winkler similarity score against each catalog entry
+3. Applies substring fallback: if compressed words match (e.g., "HandOld" → "Hand Old Bar & Grill"),
+   boosts score to 0.85
+4. Returns the highest-scoring catalog entry if score ≥ threshold, else empty string
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"]
+julia> GeneralUtils.resolve_entity("HandOld", catalog, threshold=0.5)
+"Hand Old Bar & Grill"
+
+julia> GeneralUtils.resolve_entity("Wine Cellar", catalog, threshold=0.5)
+"Wine Cellar"
+
+julia> GeneralUtils.resolve_entity("Unknown Place", catalog, threshold=0.5)
+""
+```
+"""
+function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String
+    best_match = ""
+    highest_score = 0.0
+    
+    # Normalize input text to ensure case-insensitive matching
+    clean_input = lowercase(strip(messy_input))
+    
+    for real_string in catalog
+        clean_real = lowercase(real_string)
+        
+        # Calculate phonetic/structural similarity score (0.0 to 1.0)
+        # JaroWinkler is optimized for short strings, names, and partial acronyms
+        score = compare(clean_real, clean_input, JaroWinkler())
+        
+        # Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill"
+        # We strip spaces to check if the user just compressed words together
+        if contains(replace(clean_real, " " => ""), clean_input)
+            score = max(score, 0.85) 
+        end
+        
+        if score > highest_score
+            highest_score = score
+            best_match = real_string
+        end
+    end
+    
+    # Only return if we cross our safety confidence barrier
+    if highest_score >= threshold
+        return best_match
+    end
+    
+    return "" # No confident match found
+end