ton/GeneralUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-07-13 21:22:38 +07:00
parent edad442242
commit d658d9a25b
3 changed files with 205 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Manifest.toml
+72 -8
View File
@@ -2,7 +2,7 @@
julia_version = "1.12.6"
manifest_format = "2.0"
project_hash = "c825feef41198c770952e1181ec41e9f2aa0c3c0"
project_hash = "92ca9c293aa799cfd151fbc0abb978aa6e6ee00b"
[[deps.Accessors]]
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
@@ -34,6 +34,12 @@ git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff"
uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8"
version = "1.1.3"
[[deps.ArnoldiMethod]]
deps = ["LinearAlgebra", "Random", "StaticArrays"]
git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6"
uuid = "ec485272-7323-5ecc-a04f-4719b315124d"
version = "0.4.0"
[[deps.Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
version = "1.11.0"
@@ -156,6 +162,20 @@ deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
version = "1.11.0"
[[deps.Distances]]
deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d"
uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
version = "0.10.12"
[deps.Distances.extensions]
DistancesChainRulesCoreExt = "ChainRulesCore"
DistancesSparseArraysExt = "SparseArrays"
[deps.Distances.weakdeps]
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
[[deps.Distributions]]
deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3"
@@ -207,6 +227,7 @@ deps = ["LinearAlgebra"]
git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
version = "1.16.0"
weakdeps = ["PDMats", "SparseArrays", "StaticArrays", "Statistics"]
[deps.FillArrays.extensions]
FillArraysPDMatsExt = "PDMats"
@@ -214,23 +235,30 @@ version = "1.16.0"
FillArraysStaticArraysExt = "StaticArrays"
FillArraysStatisticsExt = "Statistics"
[deps.FillArrays.weakdeps]
PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
[[deps.Future]]
deps = ["Random"]
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
version = "1.11.0"
[[deps.GeneralUtils]]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"]
path = "."
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
version = "0.4.10"
[[deps.Graphs]]
deps = ["ArnoldiMethod", "DataStructures", "Inflate", "LinearAlgebra", "Random", "SimpleTraits", "SparseArrays", "Statistics"]
git-tree-sha1 = "7eb45fe833a5b7c51cf6d89c5a841d5967e44be3"
uuid = "86223c79-3864-5bf0-83f7-82e725a168b6"
version = "1.14.0"
[deps.Graphs.extensions]
GraphsSharedArraysExt = "SharedArrays"
[deps.Graphs.weakdeps]
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
[[deps.HTTP]]
deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55"
@@ -248,6 +276,11 @@ git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec"
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
version = "0.3.28"
[[deps.Inflate]]
git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d"
uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9"
version = "0.1.5"
[[deps.InlineStrings]]
git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d"
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
@@ -628,6 +661,12 @@ version = "1.4.10"
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
version = "1.11.0"
[[deps.SimpleTraits]]
deps = ["InteractiveUtils", "MacroTools"]
git-tree-sha1 = "7ddb0b49c109481b046972c0e4ab02b2127d6a75"
uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d"
version = "0.9.6"
[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
version = "1.11.0"
@@ -661,6 +700,25 @@ version = "2.8.0"
[deps.SpecialFunctions.weakdeps]
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
[[deps.StaticArrays]]
deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"]
git-tree-sha1 = "246a8bb2e6667f832eea063c3a56aef96429a3db"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "1.9.18"
[deps.StaticArrays.extensions]
StaticArraysChainRulesCoreExt = "ChainRulesCore"
StaticArraysStatisticsExt = "Statistics"
[deps.StaticArrays.weakdeps]
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
[[deps.StaticArraysCore]]
git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a"
uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
version = "1.4.4"
[[deps.Statistics]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
@@ -697,6 +755,12 @@ version = "2.2.0"
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
[[deps.StringDistances]]
deps = ["Distances", "StatsAPI"]
git-tree-sha1 = "cd83a04baf746e3b43b83c61b7de77ab0409b80a"
uuid = "88034a9c-02f8-509d-84a9-84ec65e18404"
version = "1.0.0"
[[deps.StringManipulation]]
deps = ["PrecompileTools"]
git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
Project.toml
+4
View File
@@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
@@ -16,10 +17,13 @@ PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[compat]
Graphs = "1.14.0"
HTTP = "2.5.0 - 2.9.9"
JSON = "1.3.0 - 1.9.9"
NATS = "0.1.0"
Revise = "3.13.2"
StringDistances = "1.0.0"
src/llmUtil.jl
+129 -6
View File
@@ -1,9 +1,10 @@
module llmUtil
export formatLLMtext, extractthink,
checkAgentResponse_JSON, clean_json_response
export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
extract_vector_metadata, generate_embedding_payloads, resolve_semantic_cluster,
harvest_entity_catalog, resolve_entity
using UUIDs, JSON, Dates, DataFrames
using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs
using GeneralUtils
# ---------------------------------------------- 100 --------------------------------------------- #
@@ -228,7 +229,7 @@ Returns a DataFrame designed for vector embedding generation.
# Arguments
- `pg_conn_str::String`
PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname")
PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
# Return
- `DataFrame`
@@ -244,7 +245,7 @@ Returns a DataFrame designed for vector embedding generation.
# Example
```julia
julia> using GeneralUtils
julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
DataFrame
6 rows × 7 columns
@@ -333,7 +334,7 @@ The function constructs rich text payloads by:
# Example
```julia
julia> using GeneralUtils
julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
julia> payloads = GeneralUtils.generate_embedding_payloads(df)
3-element Vector{Dict}:
@@ -496,10 +497,132 @@ function resolve_semantic_cluster(
end
""" Harvest entity catalog from database column.
Extracts unique, non-null values from a specific column to build a local index for
semantic search or entity resolution.
# Arguments
- `conn_str::String`
PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
- `table::String`
Table name to query
- `column::String`
Column name containing entity values
# Return
- `Vector{String}`
A vector of unique, stripped strings from the specified column. Empty strings
are removed via `strip()`.
# Details
The function:
1. Connects to PostgreSQL database
2. Executes `SELECT DISTINCT column FROM table WHERE column IS NOT NULL`
3. Converts result to DataFrame
4. Strips whitespace from each value and converts to String
5. Returns clean vector of unique entity values
# Example
```julia
julia> using GeneralUtils
julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name")
["Apple", "Banana", "Orange", "Mango"]
```
"""
function harvest_entity_catalog(conn_str::String, table::String, column::String)::Vector{String}
conn = LibPQ.Connection(conn_str)
# We only care about unique, non-null values to keep the index fast and dense
query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;"
try
df = DataFrame(execute(conn, query))
# Return as a clean array of strings
return String.(strip.(df[:, 1]))
finally
close(conn)
end
end
""" Resolve entity name from messy input using fuzzy string matching.
Matches user-provided text against a reference catalog using Jaro-Winkler similarity
and returns the closest matching exact string from the database catalog.
# Arguments
- `messy_input::String`
The user input text that may contain typos, compressed words, or variations.
- `catalog::Vector{String}`
A vector of valid, exact entity strings from the database.
# Keyword Arguments
- `threshold::Float64` (default: `0.5`)
Minimum similarity score (0.0 to 1.0) required to return a match. Lower values
allow more lenient matching; higher values require closer matches.
# Return
- `String`
The exact matching string from `catalog` if similarity score ≥ threshold,
otherwise an empty string `""`.
# Details
The function:
1. Normalizes input to lowercase and strips whitespace
2. Computes Jaro-Winkler similarity score against each catalog entry
3. Applies substring fallback: if compressed words match (e.g., "HandOld" → "Hand Old Bar & Grill"),
boosts score to 0.85
4. Returns the highest-scoring catalog entry if score ≥ threshold, else empty string
# Example
```julia
julia> using GeneralUtils
julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"]
julia> GeneralUtils.resolve_entity("HandOld", catalog, threshold=0.5)
"Hand Old Bar & Grill"
julia> GeneralUtils.resolve_entity("Wine Cellar", catalog, threshold=0.5)
"Wine Cellar"
julia> GeneralUtils.resolve_entity("Unknown Place", catalog, threshold=0.5)
""
```
"""
function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String
best_match = ""
highest_score = 0.0
# Normalize input text to ensure case-insensitive matching
clean_input = lowercase(strip(messy_input))
for real_string in catalog
clean_real = lowercase(real_string)
# Calculate phonetic/structural similarity score (0.0 to 1.0)
# JaroWinkler is optimized for short strings, names, and partial acronyms
score = compare(clean_real, clean_input, JaroWinkler())
# Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill"
# We strip spaces to check if the user just compressed words together
if contains(replace(clean_real, " " => ""), clean_input)
score = max(score, 0.85)
end
if score > highest_score
highest_score = score
best_match = real_string
end
end
# Only return if we cross our safety confidence barrier
if highest_score >= threshold
return best_match
end
return "" # No confident match found
end