Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ton
|
73ec3bbb04 | ||
|
ton
|
1ad46c6e18 | ||
|
ton
|
95954249ce | ||
|
ton
|
a15630619a | ||
|
ton
|
f28405f3f1 | ||
|
ton
|
d658d9a25b | ||
|
ton
|
edad442242 | ||
|
ton
|
c56fc7366c |
+281
-40
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
julia_version = "1.12.6"
|
julia_version = "1.12.6"
|
||||||
manifest_format = "2.0"
|
manifest_format = "2.0"
|
||||||
project_hash = "c825feef41198c770952e1181ec41e9f2aa0c3c0"
|
project_hash = "a1c572efdd97cec1f432040f02784b67635102aa"
|
||||||
|
|
||||||
[[deps.Accessors]]
|
[[deps.Accessors]]
|
||||||
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
|
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
|
||||||
@@ -34,6 +34,22 @@ git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff"
|
|||||||
uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8"
|
uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8"
|
||||||
version = "1.1.3"
|
version = "1.1.3"
|
||||||
|
|
||||||
|
[[deps.ArgTools]]
|
||||||
|
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
|
||||||
|
version = "1.1.2"
|
||||||
|
|
||||||
|
[[deps.ArnoldiMethod]]
|
||||||
|
deps = ["LinearAlgebra", "Random", "StaticArrays"]
|
||||||
|
git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6"
|
||||||
|
uuid = "ec485272-7323-5ecc-a04f-4719b315124d"
|
||||||
|
version = "0.4.0"
|
||||||
|
|
||||||
|
[[deps.ArrowTypes]]
|
||||||
|
deps = ["Sockets", "UUIDs"]
|
||||||
|
git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101"
|
||||||
|
uuid = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
|
||||||
|
version = "2.3.0"
|
||||||
|
|
||||||
[[deps.Artifacts]]
|
[[deps.Artifacts]]
|
||||||
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
|
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
@@ -47,6 +63,11 @@ git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643"
|
|||||||
uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
|
uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
|
|
||||||
|
[[deps.CEnum]]
|
||||||
|
git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc"
|
||||||
|
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
|
||||||
|
version = "0.5.0"
|
||||||
|
|
||||||
[[deps.CRC32c]]
|
[[deps.CRC32c]]
|
||||||
uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
|
uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
@@ -76,9 +97,9 @@ uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
|
|||||||
version = "0.7.8"
|
version = "0.7.8"
|
||||||
|
|
||||||
[[deps.CommonSolve]]
|
[[deps.CommonSolve]]
|
||||||
git-tree-sha1 = "99ee296f88c12485402e37c2fd025f95ae097637"
|
git-tree-sha1 = "eeaad7cef88554c2fa56b5a3f71cfd5cb708c662"
|
||||||
uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
|
uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
|
||||||
version = "0.2.9"
|
version = "0.2.11"
|
||||||
|
|
||||||
[[deps.Compat]]
|
[[deps.Compat]]
|
||||||
deps = ["TOML", "UUIDs"]
|
deps = ["TOML", "UUIDs"]
|
||||||
@@ -129,6 +150,11 @@ git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
|
|||||||
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
|
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
|
||||||
version = "4.1.1"
|
version = "4.1.1"
|
||||||
|
|
||||||
|
[[deps.DBInterface]]
|
||||||
|
git-tree-sha1 = "a444404b3f94deaa43ca2a58e18153a82695282b"
|
||||||
|
uuid = "a10d1c49-ce27-4219-8d33-6db1a4562965"
|
||||||
|
version = "2.6.1"
|
||||||
|
|
||||||
[[deps.DataAPI]]
|
[[deps.DataAPI]]
|
||||||
git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
|
git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
|
||||||
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
|
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
|
||||||
@@ -156,6 +182,30 @@ deps = ["Printf"]
|
|||||||
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.Decimals]]
|
||||||
|
git-tree-sha1 = "e98abef36d02a0ec385d68cd7dadbce9b28cbd88"
|
||||||
|
uuid = "abce61dc-4473-55a0-ba07-351d65e31d42"
|
||||||
|
version = "0.4.1"
|
||||||
|
|
||||||
|
[[deps.Distances]]
|
||||||
|
deps = ["LinearAlgebra", "Statistics", "StatsAPI"]
|
||||||
|
git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d"
|
||||||
|
uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
|
||||||
|
version = "0.10.12"
|
||||||
|
|
||||||
|
[deps.Distances.extensions]
|
||||||
|
DistancesChainRulesCoreExt = "ChainRulesCore"
|
||||||
|
DistancesSparseArraysExt = "SparseArrays"
|
||||||
|
|
||||||
|
[deps.Distances.weakdeps]
|
||||||
|
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
|
||||||
|
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
|
||||||
|
|
||||||
|
[[deps.Distributed]]
|
||||||
|
deps = ["Random", "Serialization", "Sockets"]
|
||||||
|
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
|
||||||
|
version = "1.11.0"
|
||||||
|
|
||||||
[[deps.Distributions]]
|
[[deps.Distributions]]
|
||||||
deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
|
deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
|
||||||
git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3"
|
git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3"
|
||||||
@@ -179,25 +229,32 @@ git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c"
|
|||||||
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
|
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
|
||||||
version = "0.9.5"
|
version = "0.9.5"
|
||||||
|
|
||||||
|
[[deps.Downloads]]
|
||||||
|
deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
|
||||||
|
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
|
||||||
|
version = "1.7.0"
|
||||||
|
|
||||||
[[deps.EnumX]]
|
[[deps.EnumX]]
|
||||||
git-tree-sha1 = "c49898e8438c828577f04b92fc9368c388ac783c"
|
git-tree-sha1 = "c49898e8438c828577f04b92fc9368c388ac783c"
|
||||||
uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
|
uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
|
||||||
version = "1.0.7"
|
version = "1.0.7"
|
||||||
|
|
||||||
|
[[deps.ExprTools]]
|
||||||
|
git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
|
||||||
|
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
|
||||||
|
version = "0.1.10"
|
||||||
|
|
||||||
[[deps.FilePathsBase]]
|
[[deps.FilePathsBase]]
|
||||||
deps = ["Compat", "Dates"]
|
deps = ["Compat", "Dates"]
|
||||||
git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2"
|
git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2"
|
||||||
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
|
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
|
||||||
version = "0.9.24"
|
version = "0.9.24"
|
||||||
|
weakdeps = ["Mmap", "Test"]
|
||||||
|
|
||||||
[deps.FilePathsBase.extensions]
|
[deps.FilePathsBase.extensions]
|
||||||
FilePathsBaseMmapExt = "Mmap"
|
FilePathsBaseMmapExt = "Mmap"
|
||||||
FilePathsBaseTestExt = "Test"
|
FilePathsBaseTestExt = "Test"
|
||||||
|
|
||||||
[deps.FilePathsBase.weakdeps]
|
|
||||||
Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
|
|
||||||
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
|
|
||||||
|
|
||||||
[[deps.FileWatching]]
|
[[deps.FileWatching]]
|
||||||
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
|
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
@@ -207,6 +264,7 @@ deps = ["LinearAlgebra"]
|
|||||||
git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3"
|
git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3"
|
||||||
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
|
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
|
||||||
version = "1.16.0"
|
version = "1.16.0"
|
||||||
|
weakdeps = ["PDMats", "SparseArrays", "StaticArrays", "Statistics"]
|
||||||
|
|
||||||
[deps.FillArrays.extensions]
|
[deps.FillArrays.extensions]
|
||||||
FillArraysPDMatsExt = "PDMats"
|
FillArraysPDMatsExt = "PDMats"
|
||||||
@@ -214,28 +272,40 @@ version = "1.16.0"
|
|||||||
FillArraysStaticArraysExt = "StaticArrays"
|
FillArraysStaticArraysExt = "StaticArrays"
|
||||||
FillArraysStatisticsExt = "Statistics"
|
FillArraysStatisticsExt = "Statistics"
|
||||||
|
|
||||||
[deps.FillArrays.weakdeps]
|
|
||||||
PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150"
|
|
||||||
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
|
|
||||||
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
|
|
||||||
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
|
|
||||||
|
|
||||||
[[deps.Future]]
|
[[deps.Future]]
|
||||||
deps = ["Random"]
|
deps = ["Random"]
|
||||||
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
|
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.Gamma]]
|
||||||
|
git-tree-sha1 = "86f86b6168a016ed88e4ae4e64577b98c3b59e8e"
|
||||||
|
uuid = "a0844989-3bd2-4988-8bea-c9407ab0941b"
|
||||||
|
version = "1.1.0"
|
||||||
|
|
||||||
[[deps.GeneralUtils]]
|
[[deps.GeneralUtils]]
|
||||||
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "HTTP", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
|
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "Graphs", "HTTP", "JSON", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"]
|
||||||
path = "."
|
path = "."
|
||||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||||
version = "0.4.10"
|
version = "0.4.10"
|
||||||
|
|
||||||
|
[[deps.Graphs]]
|
||||||
|
deps = ["ArnoldiMethod", "DataStructures", "Inflate", "LinearAlgebra", "Random", "SimpleTraits", "SparseArrays", "Statistics"]
|
||||||
|
git-tree-sha1 = "7eb45fe833a5b7c51cf6d89c5a841d5967e44be3"
|
||||||
|
uuid = "86223c79-3864-5bf0-83f7-82e725a168b6"
|
||||||
|
version = "1.14.0"
|
||||||
|
|
||||||
|
[deps.Graphs.extensions]
|
||||||
|
GraphsSharedArraysExt = "SharedArrays"
|
||||||
|
|
||||||
|
[deps.Graphs.weakdeps]
|
||||||
|
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
|
||||||
|
SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
|
||||||
|
|
||||||
[[deps.HTTP]]
|
[[deps.HTTP]]
|
||||||
deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
|
deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
|
||||||
git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55"
|
git-tree-sha1 = "c2c808326222b6dc4bec295a83b55f79aeec98e0"
|
||||||
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||||
version = "2.5.0"
|
version = "2.5.5"
|
||||||
|
|
||||||
[[deps.HashArrayMappedTries]]
|
[[deps.HashArrayMappedTries]]
|
||||||
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
|
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
|
||||||
@@ -243,42 +313,59 @@ uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74"
|
|||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
|
|
||||||
[[deps.HypergeometricFunctions]]
|
[[deps.HypergeometricFunctions]]
|
||||||
deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
|
deps = ["Gamma", "LinearAlgebra"]
|
||||||
git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec"
|
git-tree-sha1 = "18d7deab5fb0440dc6a7b6993c5c27b25420de10"
|
||||||
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
|
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
|
||||||
version = "0.3.28"
|
version = "0.3.29"
|
||||||
|
|
||||||
|
[[deps.ICU_jll]]
|
||||||
|
deps = ["Artifacts", "JLLWrappers", "Libdl"]
|
||||||
|
git-tree-sha1 = "b3d8be712fbf9237935bde0ce9b5a736ae38fc34"
|
||||||
|
uuid = "a51ab1cf-af8e-5615-a023-bc2c838bba6b"
|
||||||
|
version = "76.2.0+0"
|
||||||
|
|
||||||
|
[[deps.Infinity]]
|
||||||
|
deps = ["Dates", "Random", "Requires"]
|
||||||
|
git-tree-sha1 = "cf8234411cbeb98676c173f930951ea29dca3b23"
|
||||||
|
uuid = "a303e19e-6eb4-11e9-3b09-cd9505f79100"
|
||||||
|
version = "0.2.4"
|
||||||
|
|
||||||
|
[[deps.Inflate]]
|
||||||
|
git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d"
|
||||||
|
uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9"
|
||||||
|
version = "0.1.5"
|
||||||
|
|
||||||
[[deps.InlineStrings]]
|
[[deps.InlineStrings]]
|
||||||
git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d"
|
git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d"
|
||||||
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
|
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
|
||||||
version = "1.4.5"
|
version = "1.4.5"
|
||||||
|
weakdeps = ["ArrowTypes", "Parsers"]
|
||||||
|
|
||||||
[deps.InlineStrings.extensions]
|
[deps.InlineStrings.extensions]
|
||||||
ArrowTypesExt = "ArrowTypes"
|
ArrowTypesExt = "ArrowTypes"
|
||||||
ParsersExt = "Parsers"
|
ParsersExt = "Parsers"
|
||||||
|
|
||||||
[deps.InlineStrings.weakdeps]
|
|
||||||
ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
|
|
||||||
Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
|
|
||||||
|
|
||||||
[[deps.InteractiveUtils]]
|
[[deps.InteractiveUtils]]
|
||||||
deps = ["Markdown"]
|
deps = ["Markdown"]
|
||||||
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
|
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.Intervals]]
|
||||||
|
deps = ["ArrowTypes", "Dates", "Printf", "RecipesBase", "Serialization", "TimeZones"]
|
||||||
|
git-tree-sha1 = "d6fe00b123e32ddd17231b35d69a6394e696fd5a"
|
||||||
|
uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
|
||||||
|
version = "1.11.0"
|
||||||
|
|
||||||
[[deps.InverseFunctions]]
|
[[deps.InverseFunctions]]
|
||||||
git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb"
|
git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb"
|
||||||
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
|
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
|
||||||
version = "0.1.17"
|
version = "0.1.17"
|
||||||
|
weakdeps = ["Dates", "Test"]
|
||||||
|
|
||||||
[deps.InverseFunctions.extensions]
|
[deps.InverseFunctions.extensions]
|
||||||
InverseFunctionsDatesExt = "Dates"
|
InverseFunctionsDatesExt = "Dates"
|
||||||
InverseFunctionsTestExt = "Test"
|
InverseFunctionsTestExt = "Test"
|
||||||
|
|
||||||
[deps.InverseFunctions.weakdeps]
|
|
||||||
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
|
||||||
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
|
|
||||||
|
|
||||||
[[deps.InvertedIndices]]
|
[[deps.InvertedIndices]]
|
||||||
git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe"
|
git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe"
|
||||||
uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
|
uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
|
||||||
@@ -289,6 +376,11 @@ git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758"
|
|||||||
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
|
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
|
||||||
version = "0.2.6"
|
version = "0.2.6"
|
||||||
|
|
||||||
|
[[deps.IterTools]]
|
||||||
|
git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023"
|
||||||
|
uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
|
||||||
|
version = "1.10.0"
|
||||||
|
|
||||||
[[deps.IteratorInterfaceExtensions]]
|
[[deps.IteratorInterfaceExtensions]]
|
||||||
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
|
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
|
||||||
uuid = "82899510-4779-5014-852e-03e436cf321d"
|
uuid = "82899510-4779-5014-852e-03e436cf321d"
|
||||||
@@ -305,25 +397,21 @@ deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs"
|
|||||||
git-tree-sha1 = "c89d196f5ffb64bfbf80985b699ea913b0d2c211"
|
git-tree-sha1 = "c89d196f5ffb64bfbf80985b699ea913b0d2c211"
|
||||||
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
||||||
version = "1.6.1"
|
version = "1.6.1"
|
||||||
|
weakdeps = ["ArrowTypes"]
|
||||||
|
|
||||||
[deps.JSON.extensions]
|
[deps.JSON.extensions]
|
||||||
JSONArrowExt = ["ArrowTypes"]
|
JSONArrowExt = ["ArrowTypes"]
|
||||||
|
|
||||||
[deps.JSON.weakdeps]
|
|
||||||
ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
|
|
||||||
|
|
||||||
[[deps.JSON3]]
|
[[deps.JSON3]]
|
||||||
deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"]
|
deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"]
|
||||||
git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975"
|
git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975"
|
||||||
uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
|
uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
|
||||||
version = "1.14.3"
|
version = "1.14.3"
|
||||||
|
weakdeps = ["ArrowTypes"]
|
||||||
|
|
||||||
[deps.JSON3.extensions]
|
[deps.JSON3.extensions]
|
||||||
JSON3ArrowExt = ["ArrowTypes"]
|
JSON3ArrowExt = ["ArrowTypes"]
|
||||||
|
|
||||||
[deps.JSON3.weakdeps]
|
|
||||||
ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
|
|
||||||
|
|
||||||
[[deps.JuliaInterpreter]]
|
[[deps.JuliaInterpreter]]
|
||||||
deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
|
deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
|
||||||
git-tree-sha1 = "58927c485919bf17ea308d9d82156de1adf4b006"
|
git-tree-sha1 = "58927c485919bf17ea308d9d82156de1adf4b006"
|
||||||
@@ -335,11 +423,32 @@ deps = ["StyledStrings"]
|
|||||||
uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"
|
uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"
|
||||||
version = "1.12.0"
|
version = "1.12.0"
|
||||||
|
|
||||||
|
[[deps.Kerberos_krb5_jll]]
|
||||||
|
deps = ["Artifacts", "JLLWrappers", "Libdl"]
|
||||||
|
git-tree-sha1 = "0f2899fdadaab4b8f57db558ba21bdb4fb52f1f0"
|
||||||
|
uuid = "b39eb1a6-c29a-53d7-8c32-632cd16f18da"
|
||||||
|
version = "1.21.3+0"
|
||||||
|
|
||||||
[[deps.LaTeXStrings]]
|
[[deps.LaTeXStrings]]
|
||||||
git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"
|
git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"
|
||||||
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
|
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
|
||||||
version = "1.4.0"
|
version = "1.4.0"
|
||||||
|
|
||||||
|
[[deps.LayerDicts]]
|
||||||
|
git-tree-sha1 = "6087ad3521d6278ebe5c27ae55e7bbb15ca312cb"
|
||||||
|
uuid = "6f188dcb-512c-564b-bc01-e0f76e72f166"
|
||||||
|
version = "1.0.0"
|
||||||
|
|
||||||
|
[[deps.LibCURL]]
|
||||||
|
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
|
||||||
|
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
|
||||||
|
version = "0.6.4"
|
||||||
|
|
||||||
|
[[deps.LibCURL_jll]]
|
||||||
|
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"]
|
||||||
|
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
|
||||||
|
version = "8.15.0+0"
|
||||||
|
|
||||||
[[deps.LibGit2]]
|
[[deps.LibGit2]]
|
||||||
deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
|
deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
|
||||||
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
|
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
|
||||||
@@ -350,6 +459,18 @@ deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"]
|
|||||||
uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
|
uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
|
||||||
version = "1.9.0+0"
|
version = "1.9.0+0"
|
||||||
|
|
||||||
|
[[deps.LibPQ]]
|
||||||
|
deps = ["CEnum", "DBInterface", "Dates", "Decimals", "DocStringExtensions", "FileWatching", "Infinity", "Intervals", "IterTools", "LayerDicts", "LibPQ_jll", "Libdl", "Memento", "OffsetArrays", "SQLStrings", "Tables", "TimeZones", "UTCDateTimes"]
|
||||||
|
git-tree-sha1 = "3d227cd13cbf1e9a54d7748dab33e078da6f9168"
|
||||||
|
uuid = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
|
||||||
|
version = "1.18.0"
|
||||||
|
|
||||||
|
[[deps.LibPQ_jll]]
|
||||||
|
deps = ["Artifacts", "ICU_jll", "JLLWrappers", "Kerberos_krb5_jll", "Libdl", "OpenSSL_jll", "Zstd_jll"]
|
||||||
|
git-tree-sha1 = "c692057e05ba6da348bc45d5dab8c7a2c88da518"
|
||||||
|
uuid = "08be9ffa-1c94-5ee5-a977-46a84ec9b350"
|
||||||
|
version = "16.14.0+0"
|
||||||
|
|
||||||
[[deps.LibSSH2_jll]]
|
[[deps.LibSSH2_jll]]
|
||||||
deps = ["Artifacts", "Libdl", "OpenSSL_jll"]
|
deps = ["Artifacts", "Libdl", "OpenSSL_jll"]
|
||||||
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
|
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
|
||||||
@@ -412,6 +533,12 @@ git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8"
|
|||||||
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
|
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
|
||||||
version = "2.28.1010+0"
|
version = "2.28.1010+0"
|
||||||
|
|
||||||
|
[[deps.Memento]]
|
||||||
|
deps = ["Dates", "Distributed", "Requires", "Serialization", "Sockets", "Test", "UUIDs"]
|
||||||
|
git-tree-sha1 = "e03a25cb3b6569623f8246d3d8b3faa7ce86f4ad"
|
||||||
|
uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9"
|
||||||
|
version = "1.5.0"
|
||||||
|
|
||||||
[[deps.Missings]]
|
[[deps.Missings]]
|
||||||
deps = ["DataAPI"]
|
deps = ["DataAPI"]
|
||||||
git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d"
|
git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d"
|
||||||
@@ -422,6 +549,12 @@ version = "1.2.0"
|
|||||||
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
|
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.Mocking]]
|
||||||
|
deps = ["Compat", "ExprTools"]
|
||||||
|
git-tree-sha1 = "2c140d60d7cb82badf06d8783800d0bcd1a7daa2"
|
||||||
|
uuid = "78c3b35d-d492-501b-9361-3d52fe80e533"
|
||||||
|
version = "0.8.1"
|
||||||
|
|
||||||
[[deps.MozillaCACerts_jll]]
|
[[deps.MozillaCACerts_jll]]
|
||||||
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
|
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
|
||||||
version = "2025.11.4"
|
version = "2025.11.4"
|
||||||
@@ -442,6 +575,17 @@ version = "1.0.3"
|
|||||||
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
|
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
|
|
||||||
|
[[deps.OffsetArrays]]
|
||||||
|
git-tree-sha1 = "117432e406b5c023f665fa73dc26e79ec3630151"
|
||||||
|
uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
|
||||||
|
version = "1.17.0"
|
||||||
|
|
||||||
|
[deps.OffsetArrays.extensions]
|
||||||
|
OffsetArraysAdaptExt = "Adapt"
|
||||||
|
|
||||||
|
[deps.OffsetArrays.weakdeps]
|
||||||
|
Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
|
||||||
|
|
||||||
[[deps.OpenBLAS_jll]]
|
[[deps.OpenBLAS_jll]]
|
||||||
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
|
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
|
||||||
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
|
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
|
||||||
@@ -509,15 +653,17 @@ version = "0.4.2"
|
|||||||
|
|
||||||
[[deps.PrettyTables]]
|
[[deps.PrettyTables]]
|
||||||
deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"]
|
deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"]
|
||||||
git-tree-sha1 = "624de6279ab7d94fc9f672f0068107eb6619732c"
|
git-tree-sha1 = "ebf455bb866ee6737030e3d3816bb6a0683c4325"
|
||||||
uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
|
uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
|
||||||
version = "3.3.2"
|
version = "3.4.0"
|
||||||
|
|
||||||
[deps.PrettyTables.extensions]
|
[deps.PrettyTables.extensions]
|
||||||
|
PrettyTablesExcelExt = "XLSX"
|
||||||
PrettyTablesTypstryExt = "Typstry"
|
PrettyTablesTypstryExt = "Typstry"
|
||||||
|
|
||||||
[deps.PrettyTables.weakdeps]
|
[deps.PrettyTables.weakdeps]
|
||||||
Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e"
|
Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e"
|
||||||
|
XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0"
|
||||||
|
|
||||||
[[deps.Printf]]
|
[[deps.Printf]]
|
||||||
deps = ["Unicode"]
|
deps = ["Unicode"]
|
||||||
@@ -551,11 +697,23 @@ deps = ["SHA"]
|
|||||||
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.RecipesBase]]
|
||||||
|
deps = ["PrecompileTools"]
|
||||||
|
git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff"
|
||||||
|
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
|
||||||
|
version = "1.3.4"
|
||||||
|
|
||||||
[[deps.Reexport]]
|
[[deps.Reexport]]
|
||||||
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
|
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
|
||||||
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
|
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
|
|
||||||
|
[[deps.Requires]]
|
||||||
|
deps = ["UUIDs"]
|
||||||
|
git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64"
|
||||||
|
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
|
||||||
|
version = "1.3.1"
|
||||||
|
|
||||||
[[deps.Reseau]]
|
[[deps.Reseau]]
|
||||||
deps = ["NetworkOptions", "OpenSSL_jll", "PrecompileTools", "Random", "SHA"]
|
deps = ["NetworkOptions", "OpenSSL_jll", "PrecompileTools", "Random", "SHA"]
|
||||||
git-tree-sha1 = "0eab6d95ed40c2ef3992255c1c71e4f9748932b5"
|
git-tree-sha1 = "0eab6d95ed40c2ef3992255c1c71e4f9748932b5"
|
||||||
@@ -567,13 +725,11 @@ deps = ["CRC32c", "CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInte
|
|||||||
git-tree-sha1 = "27e3ee13fc8739a59b380d6163d6a82f52c03bd7"
|
git-tree-sha1 = "27e3ee13fc8739a59b380d6163d6a82f52c03bd7"
|
||||||
uuid = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
uuid = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
||||||
version = "3.15.1"
|
version = "3.15.1"
|
||||||
|
weakdeps = ["Distributed"]
|
||||||
|
|
||||||
[deps.Revise.extensions]
|
[deps.Revise.extensions]
|
||||||
DistributedExt = "Distributed"
|
DistributedExt = "Distributed"
|
||||||
|
|
||||||
[deps.Revise.weakdeps]
|
|
||||||
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
|
|
||||||
|
|
||||||
[[deps.Rmath]]
|
[[deps.Rmath]]
|
||||||
deps = ["Random", "Rmath_jll"]
|
deps = ["Random", "Rmath_jll"]
|
||||||
git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b"
|
git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b"
|
||||||
@@ -588,9 +744,9 @@ version = "0.5.1+0"
|
|||||||
|
|
||||||
[[deps.Roots]]
|
[[deps.Roots]]
|
||||||
deps = ["Accessors", "CommonSolve", "Printf"]
|
deps = ["Accessors", "CommonSolve", "Printf"]
|
||||||
git-tree-sha1 = "91cfb1cb4f6e27557cc2df798a31eff6089a41eb"
|
git-tree-sha1 = "125cbd31a56de53169c3eed9c17180bc6c245f83"
|
||||||
uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665"
|
uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665"
|
||||||
version = "3.0.0"
|
version = "3.0.4"
|
||||||
|
|
||||||
[deps.Roots.extensions]
|
[deps.Roots.extensions]
|
||||||
RootsChainRulesCoreExt = "ChainRulesCore"
|
RootsChainRulesCoreExt = "ChainRulesCore"
|
||||||
@@ -612,12 +768,23 @@ version = "3.0.0"
|
|||||||
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
|
|
||||||
|
[[deps.SQLStrings]]
|
||||||
|
git-tree-sha1 = "55de0530689832b1d3d43491ee6b67bd54d3323c"
|
||||||
|
uuid = "af517c2e-c243-48fa-aab8-efac3db270f5"
|
||||||
|
version = "0.1.0"
|
||||||
|
|
||||||
[[deps.ScopedValues]]
|
[[deps.ScopedValues]]
|
||||||
deps = ["HashArrayMappedTries", "Logging"]
|
deps = ["HashArrayMappedTries", "Logging"]
|
||||||
git-tree-sha1 = "67a144433c4ce877ee6d1ada69a124d6b1ecf7be"
|
git-tree-sha1 = "67a144433c4ce877ee6d1ada69a124d6b1ecf7be"
|
||||||
uuid = "7e506255-f358-4e82-b7e4-beb19740aa63"
|
uuid = "7e506255-f358-4e82-b7e4-beb19740aa63"
|
||||||
version = "1.6.2"
|
version = "1.6.2"
|
||||||
|
|
||||||
|
[[deps.Scratch]]
|
||||||
|
deps = ["Dates"]
|
||||||
|
git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109"
|
||||||
|
uuid = "6c6a2e73-6563-6170-7368-637461726353"
|
||||||
|
version = "1.3.0"
|
||||||
|
|
||||||
[[deps.SentinelArrays]]
|
[[deps.SentinelArrays]]
|
||||||
deps = ["Dates", "Random"]
|
deps = ["Dates", "Random"]
|
||||||
git-tree-sha1 = "084c47c7c5ce5cfecefa0a98dff69eb3646b5a80"
|
git-tree-sha1 = "084c47c7c5ce5cfecefa0a98dff69eb3646b5a80"
|
||||||
@@ -628,6 +795,12 @@ version = "1.4.10"
|
|||||||
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
|
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.SimpleTraits]]
|
||||||
|
deps = ["InteractiveUtils", "MacroTools"]
|
||||||
|
git-tree-sha1 = "7ddb0b49c109481b046972c0e4ab02b2127d6a75"
|
||||||
|
uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d"
|
||||||
|
version = "0.9.6"
|
||||||
|
|
||||||
[[deps.Sockets]]
|
[[deps.Sockets]]
|
||||||
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
|
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
@@ -661,6 +834,25 @@ version = "2.8.0"
|
|||||||
[deps.SpecialFunctions.weakdeps]
|
[deps.SpecialFunctions.weakdeps]
|
||||||
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
|
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
|
||||||
|
|
||||||
|
[[deps.StaticArrays]]
|
||||||
|
deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"]
|
||||||
|
git-tree-sha1 = "246a8bb2e6667f832eea063c3a56aef96429a3db"
|
||||||
|
uuid = "90137ffa-7385-5640-81b9-e52037218182"
|
||||||
|
version = "1.9.18"
|
||||||
|
|
||||||
|
[deps.StaticArrays.extensions]
|
||||||
|
StaticArraysChainRulesCoreExt = "ChainRulesCore"
|
||||||
|
StaticArraysStatisticsExt = "Statistics"
|
||||||
|
|
||||||
|
[deps.StaticArrays.weakdeps]
|
||||||
|
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
|
||||||
|
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
|
||||||
|
|
||||||
|
[[deps.StaticArraysCore]]
|
||||||
|
git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a"
|
||||||
|
uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
|
||||||
|
version = "1.4.4"
|
||||||
|
|
||||||
[[deps.Statistics]]
|
[[deps.Statistics]]
|
||||||
deps = ["LinearAlgebra"]
|
deps = ["LinearAlgebra"]
|
||||||
git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
|
git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
|
||||||
@@ -697,6 +889,12 @@ version = "2.2.0"
|
|||||||
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
|
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
|
||||||
InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
|
InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
|
||||||
|
|
||||||
|
[[deps.StringDistances]]
|
||||||
|
deps = ["Distances", "StatsAPI"]
|
||||||
|
git-tree-sha1 = "cd83a04baf746e3b43b83c61b7de77ab0409b80a"
|
||||||
|
uuid = "88034a9c-02f8-509d-84a9-84ec65e18404"
|
||||||
|
version = "1.0.0"
|
||||||
|
|
||||||
[[deps.StringManipulation]]
|
[[deps.StringManipulation]]
|
||||||
deps = ["PrecompileTools"]
|
deps = ["PrecompileTools"]
|
||||||
git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
|
git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
|
||||||
@@ -743,6 +941,12 @@ deps = ["Dates"]
|
|||||||
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
|
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
|
||||||
version = "1.0.3"
|
version = "1.0.3"
|
||||||
|
|
||||||
|
[[deps.TZJData]]
|
||||||
|
deps = ["Artifacts"]
|
||||||
|
git-tree-sha1 = "72df96b3a595b7aab1e101eb07d2a435963a97e2"
|
||||||
|
uuid = "dc5dba14-91b3-4cab-a142-028a31da12f7"
|
||||||
|
version = "1.5.0+2025b"
|
||||||
|
|
||||||
[[deps.TableTraits]]
|
[[deps.TableTraits]]
|
||||||
deps = ["IteratorInterfaceExtensions"]
|
deps = ["IteratorInterfaceExtensions"]
|
||||||
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
|
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
|
||||||
@@ -755,6 +959,21 @@ git-tree-sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0"
|
|||||||
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
|
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
|
||||||
version = "1.13.0"
|
version = "1.13.0"
|
||||||
|
|
||||||
|
[[deps.Test]]
|
||||||
|
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
|
||||||
|
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
|
||||||
|
version = "1.11.0"
|
||||||
|
|
||||||
|
[[deps.TimeZones]]
|
||||||
|
deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"]
|
||||||
|
git-tree-sha1 = "d422301b2a1e294e3e4214061e44f338cafe18a2"
|
||||||
|
uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53"
|
||||||
|
version = "1.22.2"
|
||||||
|
weakdeps = ["RecipesBase"]
|
||||||
|
|
||||||
|
[deps.TimeZones.extensions]
|
||||||
|
TimeZonesRecipesBaseExt = "RecipesBase"
|
||||||
|
|
||||||
[[deps.TranscodingStreams]]
|
[[deps.TranscodingStreams]]
|
||||||
git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742"
|
git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742"
|
||||||
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
|
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
|
||||||
@@ -765,6 +984,12 @@ git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a"
|
|||||||
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
|
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
|
||||||
version = "1.6.1"
|
version = "1.6.1"
|
||||||
|
|
||||||
|
[[deps.UTCDateTimes]]
|
||||||
|
deps = ["Dates", "TimeZones"]
|
||||||
|
git-tree-sha1 = "4af3552bf0cf4a071bf3d14bd20023ea70f31b62"
|
||||||
|
uuid = "0f7cfa37-7abf-4834-b969-a8aa512401c2"
|
||||||
|
version = "1.6.1"
|
||||||
|
|
||||||
[[deps.UUIDs]]
|
[[deps.UUIDs]]
|
||||||
deps = ["Random", "SHA"]
|
deps = ["Random", "SHA"]
|
||||||
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||||
@@ -790,6 +1015,12 @@ deps = ["Libdl"]
|
|||||||
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
|
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
|
||||||
version = "1.3.1+2"
|
version = "1.3.1+2"
|
||||||
|
|
||||||
|
[[deps.Zstd_jll]]
|
||||||
|
deps = ["Artifacts", "JLLWrappers", "Libdl"]
|
||||||
|
git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308"
|
||||||
|
uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
|
||||||
|
version = "1.5.7+1"
|
||||||
|
|
||||||
[[deps.libblastrampoline_jll]]
|
[[deps.libblastrampoline_jll]]
|
||||||
deps = ["Artifacts", "Libdl"]
|
deps = ["Artifacts", "Libdl"]
|
||||||
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
|
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
|
||||||
@@ -800,3 +1031,13 @@ deps = ["Artifacts", "JLLWrappers", "Libdl"]
|
|||||||
git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026"
|
git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026"
|
||||||
uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8"
|
uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8"
|
||||||
version = "1.0.21+0"
|
version = "1.0.21+0"
|
||||||
|
|
||||||
|
[[deps.nghttp2_jll]]
|
||||||
|
deps = ["Artifacts", "Libdl"]
|
||||||
|
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
|
||||||
|
version = "1.64.0+1"
|
||||||
|
|
||||||
|
[[deps.p7zip_jll]]
|
||||||
|
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
|
||||||
|
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
|
||||||
|
version = "17.7.0+0"
|
||||||
|
|||||||
@@ -9,17 +9,23 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
|||||||
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
||||||
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
||||||
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
|
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
|
||||||
|
Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
|
||||||
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||||
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
||||||
|
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
|
||||||
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
||||||
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
||||||
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||||
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
||||||
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
||||||
|
StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
|
||||||
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||||
|
|
||||||
[compat]
|
[compat]
|
||||||
|
Graphs = "1.14.0"
|
||||||
HTTP = "2.5.0 - 2.9.9"
|
HTTP = "2.5.0 - 2.9.9"
|
||||||
JSON = "1.3.0 - 1.9.9"
|
JSON = "1.3.0 - 1.9.9"
|
||||||
|
LibPQ = "1.18.0"
|
||||||
NATS = "0.1.0"
|
NATS = "0.1.0"
|
||||||
Revise = "3.13.2"
|
Revise = "3.13.2"
|
||||||
|
StringDistances = "1.0.0"
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
using LibPQ, JSON, Graphs, DataFrames
|
||||||
|
config = JSON.parsefile("./appconfig.json")
|
||||||
|
host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
|
||||||
|
port = parse(Int, _port)
|
||||||
|
dbname = "winedb"
|
||||||
|
user = config["externalservice"]["sommpanion_db"]["user"]
|
||||||
|
password = config["externalservice"]["sommpanion_db"]["password"]
|
||||||
|
pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password"
|
||||||
|
db_connection = LibPQ.Connection(pg_conn_str)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
|
||||||
|
using LibPQ
|
||||||
|
using DataFrames
|
||||||
|
|
||||||
|
"""
|
||||||
|
extract_column_metadata(pg_conn_str::String) -> DataFrame
|
||||||
|
|
||||||
|
Queries PostgreSQL system catalogs to extract a rich semantic text map of every
|
||||||
|
column in the database. Returns a DataFrame designed for vector embedding generation.
|
||||||
|
"""
|
||||||
|
function extract_column_metadata(pg_conn_str::String)
|
||||||
|
conn = LibPQ.Connection(pg_conn_str)
|
||||||
|
|
||||||
|
# This direct SQL query pulls the column specifications along with column-level descriptions
|
||||||
|
query = """
|
||||||
|
SELECT
|
||||||
|
c.relname AS table_name,
|
||||||
|
a.attname AS column_name,
|
||||||
|
format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||||
|
COALESCE(d.description, '') AS column_description,
|
||||||
|
CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
|
||||||
|
CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key
|
||||||
|
FROM pg_attribute a
|
||||||
|
JOIN pg_class c ON c.oid = a.attrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
-- Join to fetch column comments/descriptions
|
||||||
|
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
|
||||||
|
-- Check if column is part of a Primary Key
|
||||||
|
LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid
|
||||||
|
AND pk.contype = 'p'
|
||||||
|
AND a.attnum = ANY(pk.conkey)
|
||||||
|
-- Check if column is part of a Foreign Key
|
||||||
|
LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid
|
||||||
|
AND fk.contype = 'f'
|
||||||
|
AND a.attnum = ANY(fk.conkey)
|
||||||
|
WHERE
|
||||||
|
n.nspname = 'public' -- Only user schemas
|
||||||
|
AND c.relkind = 'r' -- Only standard tables
|
||||||
|
AND a.attnum > 0 -- Skip system hidden columns
|
||||||
|
AND NOT a.attisdropped; -- Skip dropped columns
|
||||||
|
"""
|
||||||
|
|
||||||
|
try
|
||||||
|
# Execute and format into a clean DataFrame
|
||||||
|
result = execute(conn, query)
|
||||||
|
df = DataFrame(result)
|
||||||
|
|
||||||
|
# Create a unique document ID for each vector row
|
||||||
|
df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
|
||||||
|
|
||||||
|
return df
|
||||||
|
finally
|
||||||
|
close(conn)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
generate_embedding_payloads(df::DataFrame) -> Vector{Dict}
|
||||||
|
|
||||||
|
Transforms the metadata DataFrame into structured text strings optimal for
|
||||||
|
vector space mapping.
|
||||||
|
"""
|
||||||
|
function generate_embedding_payloads(df::DataFrame)
|
||||||
|
payloads = Dict[]
|
||||||
|
|
||||||
|
for row in eachrow(df)
|
||||||
|
# 1. Build a rich text description summarizing the column's role
|
||||||
|
text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
|
||||||
|
|
||||||
|
if row.is_primary_key
|
||||||
|
text_payload *= " [PRIMARY KEY]"
|
||||||
|
end
|
||||||
|
if row.is_foreign_key
|
||||||
|
text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
|
||||||
|
end
|
||||||
|
|
||||||
|
# Append business descriptions if they exist in the database comments
|
||||||
|
if !isempty(strip(row.column_description))
|
||||||
|
text_payload *= " | Description: $(row.column_description)"
|
||||||
|
else
|
||||||
|
text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
|
||||||
|
end
|
||||||
|
|
||||||
|
# 2. Package everything neatly to be passed to your vector store client
|
||||||
|
push!(payloads, Dict(
|
||||||
|
"id" => row.vector_id,
|
||||||
|
"text_content" => text_payload,
|
||||||
|
"metadata" => Dict(
|
||||||
|
"table" => row.table_name,
|
||||||
|
"column" => row.column_name,
|
||||||
|
"type" => row.data_type
|
||||||
|
)
|
||||||
|
))
|
||||||
|
end
|
||||||
|
|
||||||
|
return payloads
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String}
|
||||||
|
|
||||||
|
Takes a scattered array of semantically matched tables from Stage 1, navigates
|
||||||
|
the undirected network structure, and isolates the minimum interconnected subgraph
|
||||||
|
required to weave ALL hits into a single valid SQL query.
|
||||||
|
"""
|
||||||
|
function resolve_semantic_cluster(
|
||||||
|
vector_hits::Vector{String},
|
||||||
|
g::SimpleGraph,
|
||||||
|
table_to_id::Dict{String, Int},
|
||||||
|
id_to_table::Dict{Int, String}
|
||||||
|
)
|
||||||
|
# Filter out hits that don't exist in our actual database graph mapping
|
||||||
|
valid_node_ids = Int[]
|
||||||
|
for hit in vector_hits
|
||||||
|
if haskey(table_to_id, hit)
|
||||||
|
push!(valid_node_ids, table_to_id[hit])
|
||||||
|
else
|
||||||
|
@warn "Vector hit '$hit' does not map to an existing database table."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
unique!(valid_node_ids)
|
||||||
|
|
||||||
|
# Edge Case Handlers
|
||||||
|
if isempty(valid_node_ids)
|
||||||
|
return String[]
|
||||||
|
elseif length(valid_node_ids) == 1
|
||||||
|
return [id_to_table[valid_node_ids[1]]]
|
||||||
|
end
|
||||||
|
|
||||||
|
# The Isolated Subgraph Set to build our final context
|
||||||
|
schema_subgraph_nodes = Set{Int}()
|
||||||
|
|
||||||
|
# Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
|
||||||
|
anchor_node = valid_node_ids[1]
|
||||||
|
push!(schema_subgraph_nodes, anchor_node)
|
||||||
|
|
||||||
|
# Phase B: Sequentially route paths to all other semantic coordinates
|
||||||
|
for target_node in valid_node_ids[2:end]
|
||||||
|
# Skip if an earlier loop trajectory already naturally absorbed this table
|
||||||
|
if target_node in schema_subgraph_nodes
|
||||||
|
continue
|
||||||
|
end
|
||||||
|
|
||||||
|
# Calculate the shortest path tree from the CURRENT state of our subgraph
|
||||||
|
# We find the shortest path from the target back to ANY node currently in our tree
|
||||||
|
shortest_paths = dijkstra_shortest_paths(g, target_node)
|
||||||
|
|
||||||
|
# Find which node currently in our subgraph is closest to the target node
|
||||||
|
closest_subgraph_node = 0
|
||||||
|
min_distance = Inf
|
||||||
|
|
||||||
|
for subgraph_node in schema_subgraph_nodes
|
||||||
|
dist = shortest_paths.dists[subgraph_node]
|
||||||
|
if dist < min_distance
|
||||||
|
min_distance = dist
|
||||||
|
closest_subgraph_node = subgraph_node
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Reconstruct the path from the target node to the closest point on our existing tree
|
||||||
|
if closest_subgraph_node != 0
|
||||||
|
curr = closest_subgraph_node
|
||||||
|
while curr != 0
|
||||||
|
push!(schema_subgraph_nodes, curr)
|
||||||
|
curr = shortest_paths.parents[curr]
|
||||||
|
if curr == target_node
|
||||||
|
push!(schema_subgraph_nodes, target_node)
|
||||||
|
break
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Map the unique structural nodes back to clean table names
|
||||||
|
return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
|
||||||
|
end
|
||||||
|
|
||||||
|
function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String})
|
||||||
|
documents_dict = Dict("documents" => text)
|
||||||
|
payloads = [("documents", documents_dict, "dictionary")]
|
||||||
|
_, msg_envelope_json_str = msghandler.smartpack(
|
||||||
|
config["externalservice"]["servicesloadbalancer"]["nats"],
|
||||||
|
payloads;
|
||||||
|
msg_purpose="embedding",
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalservice"]["fileserver"]["url"])
|
||||||
|
|
||||||
|
reply = NATS.request(nats_conn,
|
||||||
|
config["externalservice"]["servicesloadbalancer"]["nats"],
|
||||||
|
msg_envelope_json_str, timeout=120)
|
||||||
|
incoming_env_json_str = String(reply.payload)
|
||||||
|
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||||
|
embedding_response = incoming_env["payloads"][1][2]
|
||||||
|
|
||||||
|
return embedding_response
|
||||||
|
end
|
||||||
|
|
||||||
|
nats_conn = NATS.connect(config["nats_server_info"]["url"])
|
||||||
|
|
||||||
|
# Run the extractor
|
||||||
|
metadata_df = extract_column_metadata(pg_conn_str)
|
||||||
|
embedding_ready = generate_embedding_payloads(metadata_df)
|
||||||
|
|
||||||
|
println(embedding_ready[1]["text_content"])
|
||||||
|
# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items."
|
||||||
|
|
||||||
|
# use only text content
|
||||||
|
embedding_ready_2 = [i["text_content"] for i in embedding_ready]
|
||||||
|
table_embedding = get_embedding(nats_conn, embedding_ready_2)
|
||||||
|
|
||||||
|
user_question =
|
||||||
|
"""
|
||||||
|
Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b}
|
||||||
|
"""
|
||||||
|
user_question_embedding = get_embedding(nats_conn, [user_question])
|
||||||
|
|
||||||
|
|
||||||
|
using Distances
|
||||||
|
# similarity = 1 - Distances.cosine_dist(Float64.(table_embedding["data"][1]["embedding"]),
|
||||||
|
# Float64.(user_question_embedding["data"][1]["embedding"])
|
||||||
|
# )
|
||||||
|
user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
|
||||||
|
user_question_similarity = []
|
||||||
|
for i in table_embedding["data"]
|
||||||
|
i_data = i["embedding"]
|
||||||
|
i_float = Float64.(i_data)
|
||||||
|
r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
|
||||||
|
push!(user_question_similarity, r)
|
||||||
|
end
|
||||||
|
|
||||||
|
new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
|
||||||
|
sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
|
||||||
|
|
||||||
|
# top 20 of sorted_df get this tables
|
||||||
|
vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
|
||||||
|
|
||||||
|
g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
|
||||||
|
|
||||||
|
# tables that I should put schema in LLM context
|
||||||
|
optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
function find_related_tables_for_user_question(question::String)
|
||||||
|
metadata_df = extract_column_metadata(pg_conn_str)
|
||||||
|
embedding_ready = generate_embedding_payloads(metadata_df)
|
||||||
|
|
||||||
|
# use only text content
|
||||||
|
embedding_ready_2 = [i["text_content"] for i in embedding_ready]
|
||||||
|
table_embedding = get_embedding(nats_conn, embedding_ready_2)
|
||||||
|
user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
|
||||||
|
user_question_similarity = []
|
||||||
|
for i in table_embedding["data"]
|
||||||
|
i_data = i["embedding"]
|
||||||
|
i_float = Float64.(i_data)
|
||||||
|
r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
|
||||||
|
push!(user_question_similarity, r)
|
||||||
|
end
|
||||||
|
|
||||||
|
new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
|
||||||
|
sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
|
||||||
|
|
||||||
|
#WORKING extract top 20 rows of sorted_df to get tables that related to user question
|
||||||
|
vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
|
||||||
|
|
||||||
|
g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
|
||||||
|
|
||||||
|
# tables that I should put schema in LLM context
|
||||||
|
return optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
|
||||||
|
end
|
||||||
+56
-56
@@ -235,7 +235,7 @@ Does **not** mutate the input; it always allocates new containers.
|
|||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using JSON
|
julia> using JSON, DataStructures
|
||||||
julia> d = Dict(
|
julia> d = Dict(
|
||||||
"a" => 4,
|
"a" => 4,
|
||||||
"b" => 6,
|
"b" => 6,
|
||||||
@@ -278,67 +278,67 @@ OrderedDict{String,Int} with 3 entries:
|
|||||||
"b" => 2
|
"b" => 2
|
||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
function dictify(x; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing)
|
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||||
# Dict-like objects
|
)::OrderedDict where {T<:AbstractDict}
|
||||||
if x isa AbstractDict
|
# Dict-like objects
|
||||||
out = OrderedDict{keytype, Any}()
|
out = OrderedDict{keytype, Any}()
|
||||||
|
|
||||||
# 1. Process and normalize all keys from the input dictionary
|
# 1. Process and normalize all keys from the input dictionary
|
||||||
processed_dict = OrderedDict{keytype, Any}()
|
processed_dict = OrderedDict{keytype, Any}()
|
||||||
for (k, v) in x
|
for (k, v) in x
|
||||||
if keytype === String
|
if keytype === String
|
||||||
newk = string(k)
|
newk = string(k)
|
||||||
elseif keytype === Symbol
|
elseif keytype === Symbol
|
||||||
newk = Symbol(string(k))
|
newk = Symbol(string(k))
|
||||||
else
|
else
|
||||||
newk = k
|
newk = k
|
||||||
end
|
end
|
||||||
processed_dict[newk] = dictify(v; keytype=keytype, sort_order=sort_order)
|
processed_dict[newk] = dictify(v; keytype=keytype, sort_order=sort_order)
|
||||||
end
|
end
|
||||||
|
|
||||||
# 2. If a sort order is specified, apply it
|
# 2. If a sort order is specified, apply it
|
||||||
if !isnothing(sort_order)
|
if !isnothing(sort_order)
|
||||||
# Normalize the sort_order elements to match the requested keytype
|
# Normalize the sort_order elements to match the requested keytype
|
||||||
normalized_order = map(sort_order) do tk
|
normalized_order = map(sort_order) do tk
|
||||||
if keytype === String
|
if keytype === String
|
||||||
return string(tk)
|
return string(tk)
|
||||||
elseif keytype === Symbol
|
elseif keytype === Symbol
|
||||||
return Symbol(string(tk))
|
return Symbol(string(tk))
|
||||||
else
|
else
|
||||||
return tk
|
return tk
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# First, insert keys that match the requested order
|
# First, insert keys that match the requested order
|
||||||
for target_key in normalized_order
|
for target_key in normalized_order
|
||||||
if haskey(processed_dict, target_key)
|
if haskey(processed_dict, target_key)
|
||||||
out[target_key] = processed_dict[target_key]
|
out[target_key] = processed_dict[target_key]
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Then, append any remaining keys that weren't in the sort_order
|
# Then, append any remaining keys that weren't in the sort_order
|
||||||
for (k, v) in processed_dict
|
for (k, v) in processed_dict
|
||||||
if !haskey(out, k)
|
if !haskey(out, k)
|
||||||
out[k] = v
|
out[k] = v
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
# If no sort order is given, just use the processed dict
|
# If no sort order is given, just use the processed dict
|
||||||
out = processed_dict
|
out = processed_dict
|
||||||
end
|
end
|
||||||
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
# Arrays / vectors: map elements recursively
|
|
||||||
elseif x isa AbstractArray
|
|
||||||
return [dictify(element; keytype=keytype, sort_order=sort_order) for element in x]
|
|
||||||
|
|
||||||
# Everything else: return as-is
|
|
||||||
else
|
|
||||||
return x
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||||
|
) where {T<:AbstractArray}
|
||||||
|
return [dictify(element; keytype=keytype, sort_order=sort_order) for element in x]
|
||||||
|
end
|
||||||
|
|
||||||
|
function dictify(x; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||||
|
)
|
||||||
|
return x
|
||||||
|
end
|
||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
|||||||
+522
-340
@@ -1,103 +1,13 @@
|
|||||||
module llmUtil
|
module llmUtil
|
||||||
|
|
||||||
export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection, deFormatLLMtext, extractthink,
|
export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
|
||||||
checkAgentResponse_JSON, clean_json_response
|
extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster,
|
||||||
|
harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph
|
||||||
|
|
||||||
using UUIDs, JSON, Dates
|
using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ
|
||||||
using GeneralUtils
|
using ..util
|
||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
#[PENDING] update code to use JSON
|
|
||||||
|
|
||||||
""" Convert a single chat dictionary into LLM model instruct format.
|
|
||||||
|
|
||||||
# Llama 3 instruct format example
|
|
||||||
<|begin_of_text|>
|
|
||||||
<|start_header_id|>system<|end_header_id|>
|
|
||||||
You are a helpful assistant.
|
|
||||||
<|eot_id|>
|
|
||||||
<|start_header_id|>user<|end_header_id|>
|
|
||||||
Get me an icecream.
|
|
||||||
<|eot_id|>
|
|
||||||
<|start_header_id|>assistant<|end_header_id|>
|
|
||||||
Go buy it yourself at 7-11.
|
|
||||||
<|eot_id|>
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `name::T`
|
|
||||||
message owner name e.f. "system", "user" or "assistant"
|
|
||||||
- `text::T`
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `formattedtext::String`
|
|
||||||
text formatted to model format
|
|
||||||
|
|
||||||
# Example
|
|
||||||
```jldoctest
|
|
||||||
julia> using Revise
|
|
||||||
julia> using YiemAgent
|
|
||||||
julia> d = Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",)
|
|
||||||
julia> formattedtext = YiemAgent.formatLLMtext_llama3instruct(d[:name], d[:text])
|
|
||||||
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n"
|
|
||||||
```
|
|
||||||
|
|
||||||
Signature
|
|
||||||
"""
|
|
||||||
function formatLLMtext_llama3instruct(name::T, text::T;
|
|
||||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
|
||||||
formattedtext =
|
|
||||||
if name == "system"
|
|
||||||
"""
|
|
||||||
<|start_header_id|>$name<|end_header_id|>
|
|
||||||
$text
|
|
||||||
<|eot_id|>
|
|
||||||
"""
|
|
||||||
else
|
|
||||||
"""
|
|
||||||
<|start_header_id|>$name<|end_header_id|>
|
|
||||||
$text
|
|
||||||
<|eot_id|>
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
if assistantStarter
|
|
||||||
formattedtext *=
|
|
||||||
"""
|
|
||||||
<|start_header_id|>assistant<|end_header_id|>
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
return formattedtext
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
function formatLLMtext_qwen(name::T, text::T;
|
|
||||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
|
||||||
formattedtext =
|
|
||||||
if name == "system"
|
|
||||||
"""
|
|
||||||
<|im_start|>$name
|
|
||||||
$text
|
|
||||||
<|im_end|>
|
|
||||||
"""
|
|
||||||
else
|
|
||||||
"""
|
|
||||||
<|im_start|>$name
|
|
||||||
$text
|
|
||||||
<|im_end|>
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
if assistantStarter
|
|
||||||
formattedtext *=
|
|
||||||
"""
|
|
||||||
<|im_start|>assistant
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
return formattedtext
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
function formatLLMtext_qwen3(name::T, text::T;
|
function formatLLMtext_qwen3(name::T, text::T;
|
||||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
assistantStarter::Bool=false) where {T<:AbstractString}
|
||||||
@@ -127,59 +37,6 @@ function formatLLMtext_qwen3(name::T, text::T;
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
function formatLLMtext_phi4(name::T, text::T;
|
|
||||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
|
||||||
formattedtext =
|
|
||||||
if name == "system"
|
|
||||||
"""
|
|
||||||
<|system|>
|
|
||||||
$text
|
|
||||||
<|end|>
|
|
||||||
"""
|
|
||||||
else
|
|
||||||
"""
|
|
||||||
<|assistant|>
|
|
||||||
$text
|
|
||||||
<|end|>
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
if assistantStarter
|
|
||||||
formattedtext *=
|
|
||||||
"""
|
|
||||||
<|assistant|>
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
return formattedtext
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
function formatLLMtext_granite3(name::T, text::T;
|
|
||||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
|
||||||
formattedtext =
|
|
||||||
if name == "system"
|
|
||||||
"""
|
|
||||||
<|start_of_role|>system<|end_of_role|>{$text}<|end_of_text|>
|
|
||||||
"""
|
|
||||||
else
|
|
||||||
"""
|
|
||||||
<|start_of_role|>$name<|end_of_role|>{$text}<|end_of_text|>
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
if assistantStarter
|
|
||||||
formattedtext *=
|
|
||||||
"""
|
|
||||||
<|start_of_role|>assistant<|end_of_role|>{
|
|
||||||
"""
|
|
||||||
end
|
|
||||||
|
|
||||||
return formattedtext
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
""" Convert a vector of chat message dictionaries into LLM model instruct format.
|
""" Convert a vector of chat message dictionaries into LLM model instruct format.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
@@ -194,13 +51,13 @@ end
|
|||||||
# Example
|
# Example
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using Revise
|
julia> using Revise
|
||||||
julia> using YiemAgent
|
julia> using GeneralUtils
|
||||||
julia> chatmessage = [
|
julia> chatmessage = [
|
||||||
Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",),
|
Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",),
|
||||||
Dict(:name=> "user",:text=> "list me all planets in our solar system.",),
|
Dict(:name=> "user",:text=> "list me all planets in our solar system.",),
|
||||||
Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",),
|
Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",),
|
||||||
]
|
]
|
||||||
julia> formattedtext = YiemAgent.formatLLMtext(chatmessage, "llama3instruct")
|
julia> formattedtext = GeneralUtils.formatLLMtext(chatmessage, "llama3instruct")
|
||||||
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n <|start_header_id|>user<|end_header_id|>\n list me all planets in our solar system.\n <|eot_id|>\n <|start_header_id|>assistant<|end_header_id|>\n I'm sorry. I don't know. You tell me.\n <|eot_id|>\n"
|
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n <|start_header_id|>user<|end_header_id|>\n list me all planets in our solar system.\n <|eot_id|>\n <|start_header_id|>assistant<|end_header_id|>\n I'm sorry. I don't know. You tell me.\n <|eot_id|>\n"
|
||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
@@ -237,192 +94,6 @@ function formatLLMtext(messages::Vector{Dict{Symbol, T}}, formatname::String
|
|||||||
return str
|
return str
|
||||||
end
|
end
|
||||||
|
|
||||||
""" Revert LLM-format response back into regular text.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `text::String`
|
|
||||||
The LLM formatted string to be converted.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `normalText::String`
|
|
||||||
The original plain text extracted from the given LLM-formatted string.
|
|
||||||
|
|
||||||
# Example
|
|
||||||
```jldoctest
|
|
||||||
julia> using Revise
|
|
||||||
julia> using YiemAgent
|
|
||||||
julia> response = "<|begin_of_text|>This is a sample system instruction.<|eot_id|>"
|
|
||||||
julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3")
|
|
||||||
"This is a sample system instruction."
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
function deFormatLLMtext(text::String, formatname::String; includethink::Bool=false
|
|
||||||
)::String
|
|
||||||
f =
|
|
||||||
if formatname == "granite3"
|
|
||||||
deFormatLLMtext_granite3
|
|
||||||
elseif formatname == "qwen3"
|
|
||||||
deFormatLLMtext_qwen3
|
|
||||||
else
|
|
||||||
error("$formatname template not define yet")
|
|
||||||
end
|
|
||||||
|
|
||||||
r = f(text)
|
|
||||||
result = r === nothing ? text : r
|
|
||||||
return result
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
""" Revert LLM-format response back into regular text for Granite 3 format.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `text::String`
|
|
||||||
The LLM formatted string to be converted.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `normalText::Union{Nothing, String}`
|
|
||||||
The original plain text extracted from the given LLM-formatted string.
|
|
||||||
Returns nothing if the text is not in Granite 3 format.
|
|
||||||
|
|
||||||
# Example
|
|
||||||
```jldoctest
|
|
||||||
julia> using Revise
|
|
||||||
julia> using YiemAgent
|
|
||||||
julia> response = "{This is a sample LLM response.}"
|
|
||||||
julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3")
|
|
||||||
"This is a sample LLM response."
|
|
||||||
"""
|
|
||||||
function deFormatLLMtext_granite3(text::String)::Union{Nothing, String}
|
|
||||||
# check if '{' and '}' are in the text because it's a special format for the LLM response
|
|
||||||
if contains(text, "<|im_start|>assistant")
|
|
||||||
# get the text between '{' and '}'
|
|
||||||
text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1]
|
|
||||||
return text_between_braces
|
|
||||||
elseif text[end] == '}'
|
|
||||||
text = "{$text"
|
|
||||||
text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1]
|
|
||||||
else
|
|
||||||
return nothing
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
function deFormatLLMtext_qwen3(text::String)::Union{Nothing, String}
|
|
||||||
return text
|
|
||||||
end
|
|
||||||
|
|
||||||
# function deFormatLLMtext_qwen3(text::String; includethink::Bool=false)::Union{Nothing, String}
|
|
||||||
# think = nothing
|
|
||||||
# str = nothing
|
|
||||||
|
|
||||||
# if occursin("<think>", text)
|
|
||||||
# r = GeneralUtils.extractTextBetweenString(text, "<think>", "</think>")
|
|
||||||
# if r[:success]
|
|
||||||
# think = r[:text]
|
|
||||||
# end
|
|
||||||
# str = string(split(text, "</think>")[2])
|
|
||||||
# end
|
|
||||||
|
|
||||||
# if includethink == true && occursin("<think>", text)
|
|
||||||
# result = "ModelThought: $think $str"
|
|
||||||
# return result
|
|
||||||
# elseif includethink == false && occursin("<think>", text)
|
|
||||||
# result = str
|
|
||||||
# return result
|
|
||||||
# else
|
|
||||||
# return text
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
|
|
||||||
|
|
||||||
""" Attemp to correct LLM response's incorrect JSON response.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `a::T1`
|
|
||||||
one of Yiem's agent
|
|
||||||
- `input::T2`
|
|
||||||
text to be send to virtual wine customer
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `correctjson::String`
|
|
||||||
corrected json string
|
|
||||||
|
|
||||||
# Example
|
|
||||||
```jldoctest
|
|
||||||
julia>
|
|
||||||
```
|
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
|
||||||
function jsoncorrection(config::T1, input::T2, correctJsonExample::T3;
|
|
||||||
maxattempt::Integer=3
|
|
||||||
) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
|
|
||||||
|
|
||||||
incorrectjson = deepcopy(input)
|
|
||||||
correctjson = nothing
|
|
||||||
|
|
||||||
for attempt in 1:maxattempt
|
|
||||||
try
|
|
||||||
d = copy(JSON3.read(incorrectjson))
|
|
||||||
correctjson = incorrectjson
|
|
||||||
return correctjson
|
|
||||||
catch e
|
|
||||||
@warn "Attempting to correct JSON string. Attempt $attempt"
|
|
||||||
e = """$e"""
|
|
||||||
if occursin("EOF", e)
|
|
||||||
e = split(e, "EOF")[1] * "EOF"
|
|
||||||
end
|
|
||||||
incorrectjson = deepcopy(input)
|
|
||||||
_prompt =
|
|
||||||
"""
|
|
||||||
Your goal are:
|
|
||||||
1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function.
|
|
||||||
2) Provide Corrected JSON string only. Do not provide any other info.
|
|
||||||
|
|
||||||
$correctJsonExample
|
|
||||||
|
|
||||||
Let's begin!
|
|
||||||
Given JSON string: $incorrectjson
|
|
||||||
The given JSON string failed to load previously because: $e
|
|
||||||
Corrected JSON string:
|
|
||||||
"""
|
|
||||||
|
|
||||||
# apply LLM specific instruct format
|
|
||||||
externalService = config[:externalservice][:text2textinstruct]
|
|
||||||
llminfo = externalService[:llminfo]
|
|
||||||
prompt =
|
|
||||||
if llminfo[:name] == "llama3instruct"
|
|
||||||
formatLLMtext_llama3instruct("system", _prompt)
|
|
||||||
else
|
|
||||||
error("llm model name is not defied yet $(@__LINE__)")
|
|
||||||
end
|
|
||||||
|
|
||||||
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
|
|
||||||
msgMeta = GeneralUtils.generate_msgMeta(
|
|
||||||
externalService[:mqtttopic],
|
|
||||||
senderName= "jsoncorrection",
|
|
||||||
senderId= uuid4snakecase(),
|
|
||||||
receiverName= "text2textinstruct",
|
|
||||||
mqttBroker= config[:mqttServerInfo][:broker],
|
|
||||||
mqttBrokerPort= config[:mqttServerInfo][:port],
|
|
||||||
)
|
|
||||||
|
|
||||||
outgoingMsg = Dict(
|
|
||||||
:msgMeta=> msgMeta,
|
|
||||||
:payload=> Dict(
|
|
||||||
:text=> prompt,
|
|
||||||
:kwargs=> Dict(
|
|
||||||
:max_tokens=> 512,
|
|
||||||
:stop=> ["<|eot_id|>"],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
|
|
||||||
incorrectjson = result[:response][:text]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
function extractthink(text::String)
|
function extractthink(text::String)
|
||||||
think = nothing
|
think = nothing
|
||||||
@@ -470,18 +141,18 @@ The validation logic checks:
|
|||||||
# Example
|
# Example
|
||||||
|
|
||||||
```julia
|
```julia
|
||||||
julia> using YiemAgent
|
julia> using GeneralUtils
|
||||||
julia> requiredKeys = ["wine_name", "price", "rating"]
|
julia> requiredKeys = ["wine_name", "price", "rating"]
|
||||||
julia> response = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98)
|
julia> response = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98)
|
||||||
julia> YiemAgent.checkAgentResponse_JSON(response, requiredKeys)
|
julia> GeneralUtils.checkAgentResponse_JSON(response, requiredKeys)
|
||||||
(true, nothing)
|
(true, nothing)
|
||||||
|
|
||||||
julia> response_missing = Dict("wine_name"=>"Château Margaux", "price"=>250.0)
|
julia> response_missing = Dict("wine_name"=>"Château Margaux", "price"=>250.0)
|
||||||
julia> YiemAgent.checkAgentResponse_JSON(response_missing, requiredKeys)
|
julia> GeneralUtils.checkAgentResponse_JSON(response_missing, requiredKeys)
|
||||||
(false, "rating are missing from your previous response")
|
(false, "rating are missing from your previous response")
|
||||||
|
|
||||||
julia> response_extra = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98, "extra_field"=>"data")
|
julia> response_extra = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98, "extra_field"=>"data")
|
||||||
julia> YiemAgent.checkAgentResponse_JSON(response_extra, requiredKeys)
|
julia> GeneralUtils.checkAgentResponse_JSON(response_extra, requiredKeys)
|
||||||
(false, "Your previous attempt has duplicated points according to the required response format")
|
(false, "Your previous attempt has duplicated points according to the required response format")
|
||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
@@ -506,6 +177,7 @@ function checkAgentResponse_JSON(responsedict::T1, requiredKeys::T2
|
|||||||
return (ispass, errormsg)
|
return (ispass, errormsg)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
""" Convert a plain text string containing key-value pairs into a JSON-formatted string.
|
""" Convert a plain text string containing key-value pairs into a JSON-formatted string.
|
||||||
|
|
||||||
This function takes text containing key-value pairs (typically extracted from LLM responses)
|
This function takes text containing key-value pairs (typically extracted from LLM responses)
|
||||||
@@ -551,17 +223,527 @@ end
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
""" Harvest database schema as undirected graph from PostgreSQL.
|
||||||
|
|
||||||
|
Extracts table structure and relationships from a PostgreSQL database by querying
|
||||||
|
system catalogs to build a graph representation of tables and their relationships.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `pg_conn_str::String`
|
||||||
|
PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `g::SimpleGraph`: An undirected graph where nodes represent tables and edges represent
|
||||||
|
relationships (foreign keys or shared ID column patterns).
|
||||||
|
- `id_to_table::Dict{Int, String}`: Maps numerical node IDs (1..n) to actual table names.
|
||||||
|
- `table_to_id::Dict{String, Int}`: Reverse mapping from table names to graph node IDs.
|
||||||
|
|
||||||
|
# Details
|
||||||
|
The function extracts schema information using two strategies:
|
||||||
|
1. **Explicit Foreign Keys**: Queries `pg_constraint` for actual foreign key relationships
|
||||||
|
2. **Fallback Name Matching**: Infers relationships from shared column naming patterns
|
||||||
|
(e.g., `seller_id`, `product_id` columns across tables)
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret"
|
||||||
|
julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str)
|
||||||
|
julia> vertices(g)
|
||||||
|
10
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function harvest_db_undirected_schema_graph(pg_conn_str)
|
||||||
|
conn = LibPQ.Connection(pg_conn_str)
|
||||||
|
try
|
||||||
|
# 1. Fetch all user tables
|
||||||
|
table_query = """
|
||||||
|
SELECT c.relname AS table_name, c.oid
|
||||||
|
FROM pg_class c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE n.nspname = 'public' AND c.relkind = 'r';
|
||||||
|
"""
|
||||||
|
table_df = DataFrame(execute(conn, table_query))
|
||||||
|
table_names = table_df.table_name
|
||||||
|
|
||||||
|
num_tables = length(table_names)
|
||||||
|
table_to_id = Dict{String, Int}(name => i for (i, name) in enumerate(table_names))
|
||||||
|
id_to_table = Dict{Int, String}(i => name for (i, name) in enumerate(table_names))
|
||||||
|
|
||||||
|
# CRITICAL: Use SimpleGraph (Undirected) so pathfinding can traverse both ways
|
||||||
|
g = SimpleGraph(num_tables)
|
||||||
|
|
||||||
|
# 2. Strategy A: Extract Explicit Foreign Keys
|
||||||
|
fk_query = """
|
||||||
|
SELECT
|
||||||
|
conrelid::regclass::text AS source_table,
|
||||||
|
confrelid::regclass::text AS target_table
|
||||||
|
FROM pg_constraint c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.connamespace
|
||||||
|
WHERE c.contype = 'f' AND n.nspname = 'public';
|
||||||
|
"""
|
||||||
|
fk_df = DataFrame(execute(conn, fk_query))
|
||||||
|
|
||||||
|
for row in eachrow(fk_df)
|
||||||
|
src = split(replace(row.source_table, "\"" => ""), '.')[end]
|
||||||
|
tgt = split(replace(row.target_table, "\"" => ""), '.')[end]
|
||||||
|
|
||||||
|
if haskey(table_to_id, src) && haskey(table_to_id, tgt)
|
||||||
|
add_edge!(g, table_to_id[src], table_to_id[tgt])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# 3. Strategy B: Fallback Name Matching (For missing explicit constraints)
|
||||||
|
# Fetch all columns for all tables
|
||||||
|
col_query = """
|
||||||
|
SELECT a.attname AS column_name, c.relname AS table_name
|
||||||
|
FROM pg_attribute a
|
||||||
|
JOIN pg_class c ON c.oid = a.attrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE n.nspname = 'public' AND c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped;
|
||||||
|
"""
|
||||||
|
col_df = DataFrame(execute(conn, col_query))
|
||||||
|
|
||||||
|
# Group columns by name to see who shares what fields
|
||||||
|
for gdf in groupby(col_df, :column_name)
|
||||||
|
col_name = gdf.column_name[1]
|
||||||
|
|
||||||
|
# We only infer connections on ID fields (e.g., seller_id, product_id)
|
||||||
|
# Avoid matching generic names like 'id', 'created_at', or 'name'
|
||||||
|
if endswith(lowercase(col_name), "_id") && col_name != "id"
|
||||||
|
sharing_tables = gdf.table_name
|
||||||
|
|
||||||
|
# Connect all tables that share this ID column
|
||||||
|
for i in eachindex(sharing_tables), j in (i+1):length(sharing_tables)
|
||||||
|
t1 = sharing_tables[i]
|
||||||
|
t2 = sharing_tables[j]
|
||||||
|
|
||||||
|
if haskey(table_to_id, t1) && haskey(table_to_id, t2)
|
||||||
|
add_edge!(g, table_to_id[t1], table_to_id[t2])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
return g, id_to_table, table_to_id
|
||||||
|
|
||||||
|
finally
|
||||||
|
close(conn)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
""" Extract vector metadata from PostgreSQL database.
|
||||||
|
|
||||||
|
Queries PostgreSQL system catalogs to extract column metadata including table names,
|
||||||
|
column names, data types, descriptions, and constraint information (primary/foreign keys).
|
||||||
|
Returns a DataFrame designed for vector embedding generation.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `pg_conn_str::String`
|
||||||
|
PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `DataFrame`
|
||||||
|
A DataFrame with columns:
|
||||||
|
- `table_name`: Name of the table
|
||||||
|
- `column_name`: Name of the column
|
||||||
|
- `data_type`: PostgreSQL data type with typemod
|
||||||
|
- `column_description`: Column's comment/description (empty string if none)
|
||||||
|
- `is_primary_key`: Boolean indicating if column is part of primary key
|
||||||
|
- `is_foreign_key`: Boolean indicating if column is part of foreign key
|
||||||
|
- `vector_id`: Generated unique identifier in format `col_{table_name}_{column_name}`
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
|
||||||
|
julia> df = GeneralUtils.extract_column_metadata(pg_conn)
|
||||||
|
DataFrame
|
||||||
|
6 rows × 7 columns
|
||||||
|
table_name column_name data_type column_description is_primary_key is_foreign_key vector_id
|
||||||
|
─────────────┬───────────┬───────────┬───────────────────┬───────────────┬───────────────┬────────────────────────
|
||||||
|
users id integer User ID true false col_users_id
|
||||||
|
users name text User name false false col_users_name
|
||||||
|
users email text User email false false col_users_email
|
||||||
|
products id integer Product ID true false col_products_id
|
||||||
|
products price numeric Product price false false col_products_price
|
||||||
|
products user_id integer Reference to user false true col_products_user_id
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function extract_column_metadata(pg_conn_str::String)::DataFrame
|
||||||
|
conn = LibPQ.Connection(pg_conn_str)
|
||||||
|
return extract_column_metadata(conn)
|
||||||
|
end
|
||||||
|
|
||||||
|
function extract_column_metadata(conn::LibPQ.Connection)::DataFrame
|
||||||
|
# This direct SQL query pulls the column specifications along with column-level descriptions
|
||||||
|
query = """
|
||||||
|
SELECT
|
||||||
|
c.relname AS table_name,
|
||||||
|
a.attname AS column_name,
|
||||||
|
format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||||
|
COALESCE(d.description, '') AS column_description,
|
||||||
|
CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
|
||||||
|
CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key
|
||||||
|
FROM pg_attribute a
|
||||||
|
JOIN pg_class c ON c.oid = a.attrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
-- Join to fetch column comments/descriptions
|
||||||
|
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
|
||||||
|
-- Check if column is part of a Primary Key
|
||||||
|
LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid
|
||||||
|
AND pk.contype = 'p'
|
||||||
|
AND a.attnum = ANY(pk.conkey)
|
||||||
|
-- Check if column is part of a Foreign Key
|
||||||
|
LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid
|
||||||
|
AND fk.contype = 'f'
|
||||||
|
AND a.attnum = ANY(fk.conkey)
|
||||||
|
WHERE
|
||||||
|
n.nspname = 'public' -- Only user schemas
|
||||||
|
AND c.relkind = 'r' -- Only standard tables
|
||||||
|
AND a.attnum > 0 -- Skip system hidden columns
|
||||||
|
AND NOT a.attisdropped; -- Skip dropped columns
|
||||||
|
"""
|
||||||
|
|
||||||
|
try
|
||||||
|
# Execute and format into a clean DataFrame
|
||||||
|
result = LibPQ.execute(conn, query)
|
||||||
|
df = DataFrame(result)
|
||||||
|
|
||||||
|
# Create a unique document ID for each vector row
|
||||||
|
df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
|
||||||
|
|
||||||
|
return df
|
||||||
|
finally
|
||||||
|
close(conn)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" Generate embedding payloads from vector metadata DataFrame.
|
||||||
|
|
||||||
|
Transforms the metadata DataFrame from `extract_column_metadata` into a vector of
|
||||||
|
dictionaries structured for vector embedding storage and retrieval.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `df::DataFrame`
|
||||||
|
A DataFrame with columns from `extract_column_metadata`: `table_name`, `column_name`,
|
||||||
|
`data_type`, `column_description`, `is_primary_key`, `is_foreign_key`, `vector_id`.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `Vector{Dict}`
|
||||||
|
A vector of dictionaries with keys:
|
||||||
|
- `id`: The vector ID from `vector_id`
|
||||||
|
- `text_content`: A structured text string combining table, column, type, constraint
|
||||||
|
markers, and description
|
||||||
|
- `metadata`: A dictionary containing `table`, `column`, and `type`
|
||||||
|
|
||||||
|
# Details
|
||||||
|
The function constructs rich text payloads by:
|
||||||
|
1. Building a base string with table name, column name, and data type
|
||||||
|
2. Appending constraint markers `[PRIMARY KEY]` or `[FOREIGN KEY RELATIONAL LINK]`
|
||||||
|
3. Adding column description if available, otherwise generating a default description
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
|
||||||
|
julia> df = GeneralUtils.extract_column_metadata(pg_conn)
|
||||||
|
julia> payloads = GeneralUtils.generate_embedding_payloads(df)
|
||||||
|
3-element Vector{Dict}:
|
||||||
|
Dict("id" => "col_users_id", "text_content" => "Table: users | Column: id | Type: integer [PRIMARY KEY] | Description: User ID", "metadata" => Dict("table" => "users", "column" => "id", "type" => "integer"))
|
||||||
|
Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text"))
|
||||||
|
Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text"))
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function generate_embedding_payloads(df::DataFrame)::Vector{Dict}
|
||||||
|
payloads = Dict[]
|
||||||
|
|
||||||
|
for row in eachrow(df)
|
||||||
|
# 1. Build a rich text description summarizing the column's role
|
||||||
|
text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
|
||||||
|
|
||||||
|
if row.is_primary_key
|
||||||
|
text_payload *= " [PRIMARY KEY]"
|
||||||
|
end
|
||||||
|
if row.is_foreign_key
|
||||||
|
text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
|
||||||
|
end
|
||||||
|
|
||||||
|
# Append business descriptions if they exist in the database comments
|
||||||
|
if !isempty(strip(row.column_description))
|
||||||
|
text_payload *= " | Description: $(row.column_description)"
|
||||||
|
else
|
||||||
|
text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
|
||||||
|
end
|
||||||
|
|
||||||
|
# 2. Package everything neatly to be passed to your vector store client
|
||||||
|
push!(payloads, Dict(
|
||||||
|
"id" => row.vector_id,
|
||||||
|
"text_content" => text_payload,
|
||||||
|
"metadata" => Dict(
|
||||||
|
"table" => row.table_name,
|
||||||
|
"column" => row.column_name,
|
||||||
|
"type" => row.data_type
|
||||||
|
)
|
||||||
|
))
|
||||||
|
end
|
||||||
|
|
||||||
|
return payloads
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" Resolve semantic cluster from vector hits using graph traversal.
|
||||||
|
|
||||||
|
Finds the minimum interconnected subgraph that connects all semantically matched tables
|
||||||
|
from vector search results, enabling construction of valid SQL queries across related tables.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `vector_hits::Vector{String}`
|
||||||
|
A vector of table names matched semantically from Stage 1 vector search.
|
||||||
|
- `g::SimpleGraph`
|
||||||
|
An undirected graph representing table relationships (nodes=tables, edges=foreign key relations).
|
||||||
|
- `table_to_id::Dict{String, Int}`
|
||||||
|
Mapping from table names to node IDs in the graph.
|
||||||
|
- `id_to_table::Dict{Int, String}`
|
||||||
|
Reverse mapping from node IDs to table names.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `Vector{String}`
|
||||||
|
A vector of table names representing the minimum subgraph that connects all input vector hits.
|
||||||
|
The order reflects traversal path from anchor node to connected components.
|
||||||
|
|
||||||
|
# Details
|
||||||
|
The algorithm:
|
||||||
|
1. Validates vector hits against the graph's table mapping
|
||||||
|
2. Handles edge cases: empty hits, single table (returns as-is)
|
||||||
|
3. Uses the highest-ranked vector hit as anchor node
|
||||||
|
4. For each remaining hit, finds shortest path to current subgraph using Dijkstra's algorithm
|
||||||
|
5. Builds minimal connected subgraph containing all hits
|
||||||
|
6. Returns table names in traversal order
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
julia> using GeneralUtils, Graphs
|
||||||
|
julia> g = SimpleGraph(5)
|
||||||
|
julia> add_edge!(g, 1, 2)
|
||||||
|
julia> add_edge!(g, 2, 3)
|
||||||
|
julia> add_edge!(g, 3, 4)
|
||||||
|
julia> table_to_id = Dict("users" => 1, "orders" => 2, "payments" => 3, "products" => 4, "inventory" => 5)
|
||||||
|
julia> id_to_table = Dict(1 => "users", 2 => "orders", 3 => "payments", 4 => "products", 5 => "inventory")
|
||||||
|
julia> vector_hits = ["users", "products"]
|
||||||
|
julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
|
||||||
|
["users", "orders", "products"]
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function resolve_semantic_cluster(
|
||||||
|
vector_hits::Vector{String},
|
||||||
|
g::SimpleGraph,
|
||||||
|
table_to_id::Dict{String, Int},
|
||||||
|
id_to_table::Dict{Int, String}
|
||||||
|
)::Vector{String}
|
||||||
|
# Filter out hits that don't exist in our actual database graph mapping
|
||||||
|
valid_node_ids = Int[]
|
||||||
|
for hit in vector_hits
|
||||||
|
if haskey(table_to_id, hit)
|
||||||
|
push!(valid_node_ids, table_to_id[hit])
|
||||||
|
else
|
||||||
|
@warn "Vector hit '$hit' does not map to an existing database table."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
unique!(valid_node_ids)
|
||||||
|
|
||||||
|
# Edge Case Handlers
|
||||||
|
if isempty(valid_node_ids)
|
||||||
|
return String[]
|
||||||
|
elseif length(valid_node_ids) == 1
|
||||||
|
return [id_to_table[valid_node_ids[1]]]
|
||||||
|
end
|
||||||
|
|
||||||
|
# The Isolated Subgraph Set to build our final context
|
||||||
|
schema_subgraph_nodes = Set{Int}()
|
||||||
|
|
||||||
|
# Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
|
||||||
|
anchor_node = valid_node_ids[1]
|
||||||
|
push!(schema_subgraph_nodes, anchor_node)
|
||||||
|
|
||||||
|
# Phase B: Sequentially route paths to all other semantic coordinates
|
||||||
|
for target_node in valid_node_ids[2:end]
|
||||||
|
# Skip if an earlier loop trajectory already naturally absorbed this table
|
||||||
|
if target_node in schema_subgraph_nodes
|
||||||
|
continue
|
||||||
|
end
|
||||||
|
|
||||||
|
# Calculate the shortest path tree from the CURRENT state of our subgraph
|
||||||
|
# We find the shortest path from the target back to ANY node currently in our tree
|
||||||
|
shortest_paths = dijkstra_shortest_paths(g, target_node)
|
||||||
|
|
||||||
|
# Find which node currently in our subgraph is closest to the target node
|
||||||
|
closest_subgraph_node = 0
|
||||||
|
min_distance = Inf
|
||||||
|
|
||||||
|
for subgraph_node in schema_subgraph_nodes
|
||||||
|
dist = shortest_paths.dists[subgraph_node]
|
||||||
|
if dist < min_distance
|
||||||
|
min_distance = dist
|
||||||
|
closest_subgraph_node = subgraph_node
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Reconstruct the path from the target node to the closest point on our existing tree
|
||||||
|
if closest_subgraph_node != 0
|
||||||
|
curr = closest_subgraph_node
|
||||||
|
while curr != 0
|
||||||
|
push!(schema_subgraph_nodes, curr)
|
||||||
|
curr = shortest_paths.parents[curr]
|
||||||
|
if curr == target_node
|
||||||
|
push!(schema_subgraph_nodes, target_node)
|
||||||
|
break
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Map the unique structural nodes back to clean table names
|
||||||
|
return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" Harvest entity catalog from database column.
|
||||||
|
|
||||||
|
Extracts unique, non-null values from a specific column to build a local index for
|
||||||
|
semantic search or entity resolution.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `pg_conn_str::String`
|
||||||
|
PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
|
||||||
|
- `table::String`
|
||||||
|
Table name to query
|
||||||
|
- `column::String`
|
||||||
|
Column name containing entity values
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `Vector{String}`
|
||||||
|
A vector of unique, stripped strings from the specified column. Empty strings
|
||||||
|
are removed via `strip()`.
|
||||||
|
|
||||||
|
# Details
|
||||||
|
The function:
|
||||||
|
1. Connects to PostgreSQL database
|
||||||
|
2. Executes `SELECT DISTINCT column FROM table WHERE column IS NOT NULL`
|
||||||
|
3. Converts result to DataFrame
|
||||||
|
4. Strips whitespace from each value and converts to String
|
||||||
|
5. Returns clean vector of unique entity values
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
|
||||||
|
julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name")
|
||||||
|
["Apple", "Banana", "Orange", "Mango"]
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function harvest_entity_catalog(pg_conn_str::String, table::String, column::String)::Vector{String}
|
||||||
|
conn = LibPQ.Connection(pg_conn_str)
|
||||||
|
return harvest_entity_catalog(conn, table, column)
|
||||||
|
end
|
||||||
|
|
||||||
|
function harvest_entity_catalog(conn::LibPQ.Connection, table::String, column::String)::Vector{String}
|
||||||
|
|
||||||
|
# We only care about unique, non-null values to keep the index fast and dense
|
||||||
|
query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;"
|
||||||
|
|
||||||
|
try
|
||||||
|
df = DataFrame(LibPQ.execute(conn, query))
|
||||||
|
# Return as a clean array of strings
|
||||||
|
return String.(strip.(df[:, 1]))
|
||||||
|
catch
|
||||||
|
return String[]
|
||||||
|
finally
|
||||||
|
close(conn)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" Resolve entity name from messy input using fuzzy string matching.
|
||||||
|
|
||||||
|
Matches user-provided text against a reference catalog using Jaro-Winkler similarity
|
||||||
|
and returns the closest matching exact string from the database catalog.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `messy_input::String`
|
||||||
|
The user input text that may contain typos, compressed words, or variations.
|
||||||
|
- `catalog::Vector{String}`
|
||||||
|
A vector of valid, exact entity strings from the database.
|
||||||
|
|
||||||
|
# Keyword Arguments
|
||||||
|
- `threshold::Float64` (default: `0.5`)
|
||||||
|
Minimum similarity score (0.0 to 1.0) required to return a match. Lower values
|
||||||
|
allow more lenient matching; higher values require closer matches.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `String`
|
||||||
|
The exact matching string from `catalog` if similarity score ≥ threshold,
|
||||||
|
otherwise an empty string `""`.
|
||||||
|
|
||||||
|
# Details
|
||||||
|
The function:
|
||||||
|
1. Normalizes input to lowercase and strips whitespace
|
||||||
|
2. Computes Jaro-Winkler similarity score against each catalog entry
|
||||||
|
3. Applies substring fallback: if compressed words match (e.g., "HandOld" → "Hand Old Bar & Grill"),
|
||||||
|
boosts score to 0.85
|
||||||
|
4. Returns the highest-scoring catalog entry if score ≥ threshold, else empty string
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"]
|
||||||
|
julia> GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.5)
|
||||||
|
"Hand Old Bar & Grill"
|
||||||
|
|
||||||
|
julia> GeneralUtils.resolve_entity("Wine Cellar", catalog; threshold=0.5)
|
||||||
|
"Wine Cellar"
|
||||||
|
|
||||||
|
julia> GeneralUtils.resolve_entity("Unknown Place", catalog; threshold=0.5)
|
||||||
|
""
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String
|
||||||
|
best_match = ""
|
||||||
|
highest_score = 0.0
|
||||||
|
|
||||||
|
# Normalize input text to ensure case-insensitive matching
|
||||||
|
clean_input = lowercase(strip(messy_input))
|
||||||
|
|
||||||
|
for real_string in catalog
|
||||||
|
clean_real = lowercase(real_string)
|
||||||
|
|
||||||
|
# Calculate phonetic/structural similarity score (0.0 to 1.0)
|
||||||
|
# JaroWinkler is optimized for short strings, names, and partial acronyms
|
||||||
|
score = compare(clean_real, clean_input, JaroWinkler())
|
||||||
|
|
||||||
|
# Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill"
|
||||||
|
# We strip spaces to check if the user just compressed words together
|
||||||
|
if contains(replace(clean_real, " " => ""), clean_input)
|
||||||
|
score = max(score, 0.85)
|
||||||
|
end
|
||||||
|
|
||||||
|
if score > highest_score
|
||||||
|
highest_score = score
|
||||||
|
best_match = real_string
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Only return if we cross our safety confidence barrier
|
||||||
|
if highest_score >= threshold
|
||||||
|
return best_match
|
||||||
|
end
|
||||||
|
|
||||||
|
return "" # No confident match found
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user