From d658d9a25bcaf5155a5492b52f2764650bf92c14 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 13 Jul 2026 21:22:38 +0700 Subject: [PATCH] update --- Manifest.toml | 80 ++++++++++++++++++++++++++--- Project.toml | 4 ++ src/llmUtil.jl | 135 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 205 insertions(+), 14 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 44f5eb7..3cee6c9 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "c825feef41198c770952e1181ec41e9f2aa0c3c0" +project_hash = "92ca9c293aa799cfd151fbc0abb978aa6e6ee00b" [[deps.Accessors]] deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] @@ -34,6 +34,12 @@ git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" version = "1.1.3" +[[deps.ArnoldiMethod]] +deps = ["LinearAlgebra", "Random", "StaticArrays"] +git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6" +uuid = "ec485272-7323-5ecc-a04f-4719b315124d" +version = "0.4.0" + [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" @@ -156,6 +162,20 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.12" + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + + [deps.Distances.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3" @@ -207,6 +227,7 @@ deps = ["LinearAlgebra"] git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" version = "1.16.0" +weakdeps = ["PDMats", "SparseArrays", "StaticArrays", "Statistics"] [deps.FillArrays.extensions] FillArraysPDMatsExt = "PDMats" @@ -214,23 +235,30 @@ version = "1.16.0" FillArraysStaticArraysExt = "StaticArrays" FillArraysStatisticsExt = "Statistics" - [deps.FillArrays.weakdeps] - PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" [[deps.GeneralUtils]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"] path = "." uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" version = "0.4.10" +[[deps.Graphs]] +deps = ["ArnoldiMethod", "DataStructures", "Inflate", "LinearAlgebra", "Random", "SimpleTraits", "SparseArrays", "Statistics"] +git-tree-sha1 = "7eb45fe833a5b7c51cf6d89c5a841d5967e44be3" +uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" +version = "1.14.0" + + [deps.Graphs.extensions] + GraphsSharedArraysExt = "SharedArrays" + + [deps.Graphs.weakdeps] + Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + [[deps.HTTP]] deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"] git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55" @@ -248,6 +276,11 @@ git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" version = "0.3.28" +[[deps.Inflate]] +git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d" +uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" +version = "0.1.5" + [[deps.InlineStrings]] git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" @@ -628,6 +661,12 @@ version = "1.4.10" uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "7ddb0b49c109481b046972c0e4ab02b2127d6a75" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.6" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" @@ -661,6 +700,25 @@ version = "2.8.0" [deps.SpecialFunctions.weakdeps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "246a8bb2e6667f832eea063c3a56aef96429a3db" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.18" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.4" + [[deps.Statistics]] deps = ["LinearAlgebra"] git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" @@ -697,6 +755,12 @@ version = "2.2.0" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" +[[deps.StringDistances]] +deps = ["Distances", "StatsAPI"] +git-tree-sha1 = "cd83a04baf746e3b43b83c61b7de77ab0409b80a" +uuid = "88034a9c-02f8-509d-84a9-84ec65e18404" +version = "1.0.0" + [[deps.StringManipulation]] deps = ["PrecompileTools"] git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5" diff --git a/Project.toml b/Project.toml index dd9233d..dad4a61 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" @@ -16,10 +17,13 @@ PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] +Graphs = "1.14.0" HTTP = "2.5.0 - 2.9.9" JSON = "1.3.0 - 1.9.9" NATS = "0.1.0" Revise = "3.13.2" +StringDistances = "1.0.0" diff --git a/src/llmUtil.jl b/src/llmUtil.jl index be57fba..b620cf7 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -1,9 +1,10 @@ module llmUtil -export formatLLMtext, extractthink, - checkAgentResponse_JSON, clean_json_response +export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response, + extract_vector_metadata, generate_embedding_payloads, resolve_semantic_cluster, + harvest_entity_catalog, resolve_entity -using UUIDs, JSON, Dates, DataFrames +using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs using GeneralUtils # ---------------------------------------------- 100 --------------------------------------------- # @@ -228,7 +229,7 @@ Returns a DataFrame designed for vector embedding generation. # Arguments - `pg_conn_str::String` - PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname") + PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret") # Return - `DataFrame` @@ -244,7 +245,7 @@ Returns a DataFrame designed for vector embedding generation. # Example ```julia julia> using GeneralUtils -julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb" +julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" julia> df = GeneralUtils.extract_vector_metadata(pg_conn) DataFrame 6 rows × 7 columns @@ -333,7 +334,7 @@ The function constructs rich text payloads by: # Example ```julia julia> using GeneralUtils -julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb" +julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" julia> df = GeneralUtils.extract_vector_metadata(pg_conn) julia> payloads = GeneralUtils.generate_embedding_payloads(df) 3-element Vector{Dict}: @@ -496,10 +497,132 @@ function resolve_semantic_cluster( end +""" Harvest entity catalog from database column. + +Extracts unique, non-null values from a specific column to build a local index for +semantic search or entity resolution. + +# Arguments +- `conn_str::String` + PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret") +- `table::String` + Table name to query +- `column::String` + Column name containing entity values + +# Return +- `Vector{String}` + A vector of unique, stripped strings from the specified column. Empty strings + are removed via `strip()`. + +# Details +The function: +1. Connects to PostgreSQL database +2. Executes `SELECT DISTINCT column FROM table WHERE column IS NOT NULL` +3. Converts result to DataFrame +4. Strips whitespace from each value and converts to String +5. Returns clean vector of unique entity values + +# Example +```julia +julia> using GeneralUtils +julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret" +julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name") +["Apple", "Banana", "Orange", "Mango"] +``` +""" +function harvest_entity_catalog(conn_str::String, table::String, column::String)::Vector{String} + conn = LibPQ.Connection(conn_str) + + # We only care about unique, non-null values to keep the index fast and dense + query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;" + + try + df = DataFrame(execute(conn, query)) + # Return as a clean array of strings + return String.(strip.(df[:, 1])) + finally + close(conn) + end +end +""" Resolve entity name from messy input using fuzzy string matching. +Matches user-provided text against a reference catalog using Jaro-Winkler similarity +and returns the closest matching exact string from the database catalog. +# Arguments +- `messy_input::String` + The user input text that may contain typos, compressed words, or variations. +- `catalog::Vector{String}` + A vector of valid, exact entity strings from the database. + +# Keyword Arguments +- `threshold::Float64` (default: `0.5`) + Minimum similarity score (0.0 to 1.0) required to return a match. Lower values + allow more lenient matching; higher values require closer matches. + +# Return +- `String` + The exact matching string from `catalog` if similarity score ≥ threshold, + otherwise an empty string `""`. + +# Details +The function: +1. Normalizes input to lowercase and strips whitespace +2. Computes Jaro-Winkler similarity score against each catalog entry +3. Applies substring fallback: if compressed words match (e.g., "HandOld" → "Hand Old Bar & Grill"), + boosts score to 0.85 +4. Returns the highest-scoring catalog entry if score ≥ threshold, else empty string + +# Example +```julia +julia> using GeneralUtils +julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"] +julia> GeneralUtils.resolve_entity("HandOld", catalog, threshold=0.5) +"Hand Old Bar & Grill" + +julia> GeneralUtils.resolve_entity("Wine Cellar", catalog, threshold=0.5) +"Wine Cellar" + +julia> GeneralUtils.resolve_entity("Unknown Place", catalog, threshold=0.5) +"" +``` +""" +function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String + best_match = "" + highest_score = 0.0 + + # Normalize input text to ensure case-insensitive matching + clean_input = lowercase(strip(messy_input)) + + for real_string in catalog + clean_real = lowercase(real_string) + + # Calculate phonetic/structural similarity score (0.0 to 1.0) + # JaroWinkler is optimized for short strings, names, and partial acronyms + score = compare(clean_real, clean_input, JaroWinkler()) + + # Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill" + # We strip spaces to check if the user just compressed words together + if contains(replace(clean_real, " " => ""), clean_input) + score = max(score, 0.85) + end + + if score > highest_score + highest_score = score + best_match = real_string + end + end + + # Only return if we cross our safety confidence barrier + if highest_score >= threshold + return best_match + end + + return "" # No confident match found +end