diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..bafa6ab --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "julia", + "request": "launch", + "name": "Run active Julia file", + "program": "${file}", + "stopOnEntry": false, + "cwd": "${workspaceFolder}", + "env": {}, + "juliaEnv": "${command:activeJuliaEnvironment}" + } + ] +} \ No newline at end of file diff --git a/Manifest.toml b/Manifest.toml index 4e2411a..b680be7 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.4" +julia_version = "1.11.0" manifest_format = "2.0" -project_hash = "aa95ae71bc02fb7f42c7111a2492e1a213aafbf5" +project_hash = "dbd62da0dcca1a1b2302848e770ef42c10a9d0d8" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -12,13 +12,15 @@ version = "1.1.3" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.1" +version = "1.1.2" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" [[deps.BitFlags]] git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" @@ -91,10 +93,10 @@ uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.16.0" [[deps.DataFrames]] -deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "04c738083f29f86e62c8afc341f0967d8717bdb8" +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.6.1" +version = "1.7.0" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] @@ -110,35 +112,23 @@ version = "1.0.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" [[deps.Decimals]] git-tree-sha1 = "e98abef36d02a0ec385d68cd7dadbce9b28cbd88" uuid = "abce61dc-4473-55a0-ba07-351d65e31d42" version = "0.4.1" -[[deps.DispatchDoctor]] -deps = ["MacroTools", "Preferences"] -git-tree-sha1 = "c2acd1de2c4c357928f9fb6b60b402d914621378" -uuid = "8d63f2c5-f18a-4cf2-ba9d-b3f60fc568c8" -version = "0.4.14" - - [deps.DispatchDoctor.extensions] - DispatchDoctorChainRulesCoreExt = "ChainRulesCore" - DispatchDoctorEnzymeCoreExt = "EnzymeCore" - - [deps.DispatchDoctor.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" - [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] -git-tree-sha1 = "e6c693a0e4394f8fda0e51a5bdf5aef26f8235e9" +git-tree-sha1 = "d7477ecdafb813ddee2ae727afa94e9dcb5f3fb0" uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.111" +version = "0.25.112" [deps.Distributions.extensions] DistributionsChainRulesCoreExt = "ChainRulesCore" @@ -174,9 +164,9 @@ version = "0.1.10" [[deps.FileIO]] deps = ["Pkg", "Requires", "UUIDs"] -git-tree-sha1 = "82d8afa92ecf4b52d78d869f038ebfb881267322" +git-tree-sha1 = "62ca0547a14c57e98154423419d8a342dca75ca9" uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.16.3" +version = "1.16.4" [[deps.FilePathsBase]] deps = ["Compat", "Dates"] @@ -191,12 +181,13 @@ weakdeps = ["Mmap", "Test"] [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" [[deps.FillArrays]] deps = ["LinearAlgebra"] -git-tree-sha1 = "fd0002c0b5362d7eb952450ad5eb742443340d6e" +git-tree-sha1 = "6a70198746448456524cb442b8af316927ff3e1a" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.12.0" +version = "1.13.0" weakdeps = ["PDMats", "SparseArrays", "Statistics"] [deps.FillArrays.extensions] @@ -207,9 +198,10 @@ weakdeps = ["PDMats", "SparseArrays", "Statistics"] [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" [[deps.GeneralUtils]] -deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON3", "MQTTClient", "Random", "Revise", "UUIDs"] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON3", "MQTTClient", "PrettyPrinting", "Random", "SHA", "UUIDs"] path = "/appfolder/app/privatejuliapkg/GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" version = "0.1.0" @@ -254,6 +246,7 @@ version = "1.4.2" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" [[deps.Intervals]] deps = ["Dates", "Printf", "RecipesBase", "Serialization", "TimeZones"] @@ -283,9 +276,9 @@ version = "1.0.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.5.0" +version = "1.6.1" [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] @@ -301,9 +294,9 @@ version = "1.14.0" [[deps.JuliaInterpreter]] deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] -git-tree-sha1 = "4b415b6cccb9ab61fec78a621572c82ac7fa5776" +git-tree-sha1 = "2984284a8abcfcc4784d95a9e2ea4e352dd8ede7" uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" -version = "0.9.35" +version = "0.9.36" [[deps.Kerberos_krb5_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -330,6 +323,7 @@ version = "1.0.0" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] @@ -339,16 +333,17 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.4.0+0" +version = "8.6.0+0" [[deps.LibGit2]] deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" [[deps.LibGit2_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.6.4+0" +version = "1.7.2+0" [[deps.LibPQ]] deps = ["CEnum", "DBInterface", "Dates", "Decimals", "DocStringExtensions", "FileWatching", "Infinity", "Intervals", "IterTools", "LayerDicts", "LibPQ_jll", "Libdl", "Memento", "OffsetArrays", "SQLStrings", "Tables", "TimeZones", "UTCDateTimes"] @@ -369,10 +364,12 @@ version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" [[deps.LogExpFunctions]] deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] @@ -392,6 +389,7 @@ version = "0.3.28" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] @@ -401,9 +399,9 @@ version = "1.0.3" [[deps.LoweredCodeUtils]] deps = ["JuliaInterpreter"] -git-tree-sha1 = "1ce1834f9644a8f7c011eb0592b7fd6c42c90653" +git-tree-sha1 = "96d2a4a668f5c098fb8a26ce7da53cde3e462a80" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" -version = "3.0.1" +version = "3.0.3" [[deps.MQTTClient]] deps = ["Distributed", "Random", "Sockets"] @@ -424,6 +422,7 @@ version = "0.5.13" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" [[deps.MbedTLS]] deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] @@ -434,7 +433,7 @@ version = "1.1.9" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+1" +version = "2.28.6+0" [[deps.Memento]] deps = ["Dates", "Distributed", "Requires", "Serialization", "Sockets", "Test", "UUIDs"] @@ -456,6 +455,7 @@ version = "1.2.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" [[deps.Mocking]] deps = ["Compat", "ExprTools"] @@ -465,7 +465,7 @@ version = "0.8.1" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.1.10" +version = "2023.12.12" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -485,7 +485,7 @@ version = "1.14.1" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.23+4" +version = "0.3.27+1" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] @@ -500,9 +500,9 @@ version = "1.4.3" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "a028ee3cb5641cccc4c24e90c36b0a4f7707bdf5" +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.14+0" +version = "3.0.15+1" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] @@ -534,9 +534,13 @@ uuid = "fa939f87-e72e-5be4-a000-7fc836dbe307" version = "1.3.0" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.10.0" +version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" [[deps.PooledArrays]] deps = ["DataAPI", "Future"] @@ -563,18 +567,19 @@ version = "0.4.2" [[deps.PrettyTables]] deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "66b20dd35966a748321d3b2537c4584cf40387c7" +git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.3.2" +version = "2.4.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" [[deps.PtrArrays]] -git-tree-sha1 = "f011fbb92c4d401059b2212c05c0601b70f8b759" +git-tree-sha1 = "77a42d78b6a92df47ab37e177b2deac405e1c88f" uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.2.0" +version = "1.2.1" [[deps.PythonCall]] deps = ["CondaPkg", "Dates", "Libdl", "MacroTools", "Markdown", "Pkg", "REPL", "Requires", "Serialization", "Tables", "UnsafePointers"] @@ -584,17 +589,25 @@ version = "0.9.23" [[deps.QuadGK]] deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "e237232771fdafbae3db5c31275303e056afaa9f" +git-tree-sha1 = "cda3b045cf9ef07a08ad46731f5a3165e56cf3da" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.10.1" +version = "2.11.1" + + [deps.QuadGK.extensions] + QuadGKEnzymeExt = "Enzyme" + + [deps.QuadGK.weakdeps] + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" [[deps.REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" [[deps.RecipesBase]] deps = ["PrecompileTools"] @@ -615,21 +628,21 @@ version = "1.3.0" [[deps.Revise]] deps = ["CodeTracking", "Distributed", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] -git-tree-sha1 = "7b7850bb94f75762d567834d7e9802fc22d62f9c" +git-tree-sha1 = "2d4e5de3ac1c348fd39ddf8adbef82aa56b65576" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" -version = "3.5.18" +version = "3.6.1" [[deps.Rmath]] deps = ["Random", "Rmath_jll"] -git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +git-tree-sha1 = "852bd0f55565a9e973fcfee83a84413270224dc4" uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" -version = "0.7.1" +version = "0.8.0" [[deps.Rmath_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e60724fd3beea548353984dc61c943ecddb0e29a" +git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" -version = "0.4.3+0" +version = "0.5.1+0" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" @@ -654,14 +667,16 @@ version = "1.4.5" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" [[deps.SimpleBufferStream]] -git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" -version = "1.1.0" +version = "1.2.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" [[deps.SortingAlgorithms]] deps = ["DataStructures"] @@ -672,7 +687,7 @@ version = "1.2.1" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.10.0" +version = "1.11.0" [[deps.SpecialFunctions]] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] @@ -687,9 +702,14 @@ version = "2.4.0" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" [[deps.Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.10.0" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] [[deps.StatsAPI]] deps = ["LinearAlgebra"] @@ -705,9 +725,9 @@ version = "0.34.3" [[deps.StatsFuns]] deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] -git-tree-sha1 = "cef0472124fab0695b58ca35a77c6fb942fdab8a" +git-tree-sha1 = "b423576adc27097764a90e163157bcfc9acf0f46" uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "1.3.1" +version = "1.3.2" [deps.StatsFuns.extensions] StatsFunsChainRulesCoreExt = "ChainRulesCore" @@ -719,15 +739,19 @@ version = "1.3.1" [[deps.StringManipulation]] deps = ["PrecompileTools"] -git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" +git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.3.4" +version = "0.4.0" [[deps.StructTypes]] deps = ["Dates", "UUIDs"] -git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" -version = "1.10.0" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" [[deps.SuiteSparse]] deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] @@ -736,7 +760,7 @@ uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.2.1+1" +version = "7.7.0+0" [[deps.TOML]] deps = ["Dates"] @@ -745,9 +769,9 @@ version = "1.0.3" [[deps.TZJData]] deps = ["Artifacts"] -git-tree-sha1 = "1607ad46cf8d642aa779a1d45af1c8620dbf6915" +git-tree-sha1 = "36b40607bf2bf856828690e097e1c799623b0602" uuid = "dc5dba14-91b3-4cab-a142-028a31da12f7" -version = "1.2.0+2024a" +version = "1.3.0+2024b" [[deps.TableTraits]] deps = ["IteratorInterfaceExtensions"] @@ -769,21 +793,22 @@ version = "1.10.0" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" [[deps.TimeZones]] deps = ["Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"] -git-tree-sha1 = "b92aebdd3555f3a7e3267cf17702033c2814ef48" +git-tree-sha1 = "8323074bc977aa85cf5ad71099a83ac75b0ac107" uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" -version = "1.18.0" +version = "1.18.1" weakdeps = ["RecipesBase"] [deps.TimeZones.extensions] TimeZonesRecipesBaseExt = "RecipesBase" [[deps.TranscodingStreams]] -git-tree-sha1 = "e84b3a11b9bece70d14cce63406bbc79ed3464d2" +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.11.2" +version = "0.11.3" [[deps.URIs]] git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" @@ -799,9 +824,11 @@ version = "1.6.1" [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" [[deps.UnsafePointers]] git-tree-sha1 = "c81331b3b2e60a982be57c046ec91f599ede674a" @@ -826,14 +853,14 @@ version = "1.2.13+1" [[deps.Zstd_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e678132f07ddb5bfa46857f0d7620fb9be675d3b" +git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" -version = "1.5.6+0" +version = "1.5.6+1" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.8.0+1" +version = "5.11.0+0" [[deps.micromamba_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] @@ -844,7 +871,7 @@ version = "1.5.8+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.52.0+1" +version = "1.59.0+0" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] diff --git a/Project.toml b/Project.toml index 6b4529f..01c7b46 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,6 @@ CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -DispatchDoctor = "8d63f2c5-f18a-4cf2-ba9d-b3f60fc568c8" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" diff --git a/config.json b/config.json index 95c505a..69a93d0 100644 --- a/config.json +++ b/config.json @@ -17,21 +17,15 @@ "description": "a central agent server's topic to get this agent config" }, "servicetopic": { - "mqtttopic": "/yiem_branch_1/agent/sommelier/backend/prompt/api/v1.1/testing", + "mqtttopic": [ + "/yiem/branch_1/agent/wine/backend/prompt/api_v1/testing" + ], "description": "a topic this agent are waiting for service request" }, - "serviceInternalTopic": { - "mqtttopic": "/yiem_branch_1/agent/sommelier/backend/internal/api/v1.1/testing", - "description": "a topic for this agent's internal communication" - }, "role": { "value": "sommelier", "description": "agent role" }, - "keepalivetopic": { - "mqtttopic": "/yiem_branch_1/agent/sommelier/keepalive", - "description": "topic used for keepalive function" - }, "organization": { "value": "yiem_branch_1", "description": "organization name" diff --git a/core b/core new file mode 100644 index 0000000..a880739 Binary files /dev/null and b/core differ diff --git a/src/interface.jl b/src/interface.jl index 80c4e31..f38a3aa 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -250,8 +250,7 @@ function decisionMaker(state::T1, context, text2textInstructLLM::Function, error("DecisionMaker has more than one key per categories") end end - println("--> SQLLLM decisionMaker() ", @__FILE__, " ", @__LINE__) - pprintln(Dict(responsedict)) + return responsedict catch e io = IOBuffer() @@ -499,7 +498,7 @@ julia> function evaluator(state::T1, text2textInstructLLM::Function; addSQLVectorDB::Union{Function, Nothing}=nothing ) where {T1<:AbstractDict} - println("Evaluating state", @__FILE__, " ", @__LINE__) + # systemmsg = # """ # You are a helpful assistant that analyzes agent's trajectories to find solutions and observations (i.e., the results of actions) to answer the user's questions. @@ -737,14 +736,16 @@ function evaluator(state::T1, text2textInstructLLM::Function; # mark as terminal state when the answer is achieved if accepted_as_answer == "Yes" state[:isterminal] = true - state[:reward] = 1 + + # user score as reward because different answers hold different value for the user. + state[:reward] = responsedict[:score] #add to vectorDB if addSQLVectorDB !== nothing addSQLVectorDB(state) end end - println("--> 5 Evaluator ", @__FILE__, " ", @__LINE__) + println("~~~ 5 Evaluator() ", @__FILE__, " ", @__LINE__) pprintln(Dict(responsedict)) return responsedict[:score] @@ -953,7 +954,7 @@ julia> state = Dict( # TODO - [] add embedding of newstate and store in newstate[:embedding] - + - [WORKING] should getdata() return isterminal? # Signature """ function transition(state::T, args::NamedTuple @@ -992,17 +993,13 @@ function transition(state::T, args::NamedTuple else error("undefined LLM function. Requesting $actionname") end - # this section allow LLM functions above to have different return values. - result = haskey(response, :result) ? response[:result] : nothing + success::Bool = haskey(response, :success) ? response[:success] : false + result = success ? response[:result] : response[:errormsg] select = haskey(response, :select) ? response[:select] : nothing reward::Integer = haskey(response, :reward) ? response[:reward] : 0 isterminal::Bool = haskey(response, :isterminal) ? response[:isterminal] : false - errormsg::Union{AbstractString, Nothing} = haskey(response, :errormsg) ? response[:errormsg] : nothing - success::Bool = haskey(response, :success) ? response[:success] : false - newNodeKey, newstate = makeNewState(state, thoughtDict, JSON3.write(result), select, reward, isterminal) - println("SQLLLM transition() 1 ", @__FILE__, " ", @__LINE__) progressvalue::Integer = evaluatorF(newstate, text2textInstructLLM; addSQLVectorDB=addSQLVectorDBF) @@ -1090,14 +1087,14 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; addSQLVectorDB::Union{Function, Nothing}=nothing, querySQLVectorDB::Union{Function, Nothing}=nothing )::String where {T<:AbstractString} - #[WORKING] add extra context for Evaluator so that it knows the observation is from seaching a database + # add extra context for Evaluator so that it knows the observation is from seaching a database query = "Search the database for {$query}" initialstate = Dict{Symbol, Any}( :reward=> 0, :isterminal=> false, :evaluation=> "None", - :suggestion=> "None", :evaluationscore=> 0, + :suggestion=> "None", :accepted_as_answer=> "No", :lesson=> nothing, @@ -1121,10 +1118,13 @@ function query(query::T, executeSQL::Function, text2textInstructLLM::Function; addSQLVectorDB=addSQLVectorDB, ) - _, result = LLMMCTS.runMCTS(initialstate, transition, transitionargs; - totalsample=1, maxdepth=3, maxiterations=1, explorationweight=1.0) - latestKey, _ = GeneralUtils.findHighestIndexKey(result[:thoughtHistory], "observation") - resulttext = result[:thoughtHistory][latestKey] + earlystop(state) = state[:reward] >= 8 ? true : false + + _, resultState = LLMMCTS.runMCTS(initialstate, transition, transitionargs; + totalsample=1, maxdepth=3, maxiterations=3, explorationweight=1.0, + earlystop=earlystop) + latestKey, _ = GeneralUtils.findHighestIndexKey(resultState[:thoughtHistory], "observation") + resulttext = resultState[:thoughtHistory][latestKey] return resulttext end diff --git a/src/llmfunction.jl b/src/llmfunction.jl index 0c55e2e..14c0e5b 100644 --- a/src/llmfunction.jl +++ b/src/llmfunction.jl @@ -1,10 +1,10 @@ module llmfunction -export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox, - getTableNameFromSQL, extractContent_dataframe, SQLexecution +export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox, + getTableNameFromSQL, extractContent_dataframe, SQLexecution -using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV, - DataStructures, StatsBase +using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV, + DataStructures, StatsBase using GeneralUtils, LLMMCTS using ..util @@ -36,25 +36,24 @@ julia> result = response[:result] # Signature """ function listAllTable_json(executeSQL::Function - )::NamedTuple{(:result, :success), Tuple{DataFrame, Bool}} +)::NamedTuple{(:result, :success),Tuple{DataFrame,Bool}} - sql = - """ - SELECT - table_name, - obj_description(relfilenode, 'pg_class') AS table_comment, - string_agg(column_name || ' (' || data_type || ')', ', ') AS columns - FROM - information_schema.columns - JOIN - pg_class ON table_name = relname - WHERE - table_schema = 'public' - GROUP BY - table_name, relfilenode - ORDER BY - table_name; - """ + sql = """ + SELECT + table_name, + obj_description(relfilenode, 'pg_class') AS table_comment, + string_agg(column_name || ' (' || data_type || ')', ', ') AS columns + FROM + information_schema.columns + JOIN + pg_class ON table_name = relname + WHERE + table_schema = 'public' + GROUP BY + table_name, relfilenode + ORDER BY + table_name; + """ result = executeSQL(sql) df = DataFrame(result) @@ -65,24 +64,23 @@ end function listAllTable_str(executeSQL::Function - )::NamedTuple{(:result, :success), Tuple{String, Bool}} - sql = - """ - SELECT - table_name, - obj_description(relfilenode, 'pg_class') AS table_comment, - string_agg(column_name || ' (' || data_type || ')', ', ') AS columns - FROM - information_schema.columns - JOIN - pg_class ON table_name = relname - WHERE - table_schema = 'public' - GROUP BY - table_name, relfilenode - ORDER BY - table_name; - """ +)::NamedTuple{(:result, :success),Tuple{String,Bool}} + sql = """ + SELECT + table_name, + obj_description(relfilenode, 'pg_class') AS table_comment, + string_agg(column_name || ' (' || data_type || ')', ', ') AS columns + FROM + information_schema.columns + JOIN + pg_class ON table_name = relname + WHERE + table_schema = 'public' + GROUP BY + table_name, relfilenode + ORDER BY + table_name; + """ result = executeSQL(sql) df = DataFrame(result) tableinfo = "Here are a list of available tables in the database (each row is in this format: table name; table comment; table columns): \n" @@ -109,20 +107,19 @@ end """ -function tableinfo_str(executeSQL::Function, tablename::String)::NamedTuple{(:result, :success), Tuple{String, Bool}} - - sql = - """ - SELECT - column_name, - data_type, - col_description(format('%s.%s', table_schema, table_name)::regclass::oid, ordinal_position) AS column_comment - FROM - information_schema.columns - WHERE - table_name = '$tablename' - AND table_schema = 'public'; - """ +function tableinfo_str(executeSQL::Function, tablename::String)::NamedTuple{(:result, :success),Tuple{String,Bool}} + + sql = """ + SELECT + column_name, + data_type, + col_description(format('%s.%s', table_schema, table_name)::regclass::oid, ordinal_position) AS column_comment + FROM + information_schema.columns + WHERE + table_name = '$tablename' + AND table_schema = 'public'; + """ result = executeSQL(sql) df = DataFrame(result) @@ -167,18 +164,17 @@ julia> result = response[:result] # Signature """ function tableinfo(executeSQL::Function, tablenames::T - )::NamedTuple{(:result,), Tuple{String}} where {T<:AbstractVector} +)::NamedTuple{(:result,),Tuple{String}} where {T<:AbstractVector} # list all tables in a database - sql = - """ - SELECT pg_namespace.nspname AS schema_name, - relname AS table_name, - pg_catalog.obj_description(pg_class.oid) AS comment - FROM pg_class - INNER JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace - WHERE pg_namespace.nspname = 'public' -- Replace 'public' with your desired schema - AND pg_class.relkind IN ('r', 't'); - """ + sql = """ + SELECT pg_namespace.nspname AS schema_name, + relname AS table_name, + pg_catalog.obj_description(pg_class.oid) AS comment + FROM pg_class + INNER JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace + WHERE pg_namespace.nspname = 'public' -- Replace 'public' with your desired schema + AND pg_class.relkind IN ('r', 't'); + """ _result = executeSQL(sql) df = DataFrame(_result) @@ -193,8 +189,7 @@ function tableinfo(executeSQL::Function, tablenames::T end end if !isempty(notExistingTable) - result = - "Error, the following tables does not exist in the database: $(JSON3.write(notExistingTable))" + result = "Error, the following tables does not exist in the database: $(JSON3.write(notExistingTable))" return (result=result,) end @@ -221,53 +216,70 @@ end A connection object connected to the database - `text2textInstructLLM::Function` A function that handles communication to LLM service. - + # Return - `NamedTuple{(:result, :errormsg, success), Tuple{String, String, Bool}}` + +# TODO + - [x] getdata directly using sql execute # Signature """ -function getdata(query::T, context::Union{Dict, Nothing}, executeSQL::Function, +function getdata(query::T, context::Union{Dict,Nothing}, executeSQL::Function, text2textInstructLLM::Function; - )::NamedTuple{(:result, :errormsg, :success), Tuple{String, Union{String, Nothing}, Bool}} where {T<:AbstractString} +) where {T<:AbstractString} - # get table info here because it'll be called only 1-time. If this function is in - # getdata_decisionMaker(), it'll be called everytime - mentionedtable = getTableNameFromSQL(query, text2textInstructLLM) - mentionedTableInfo = tableinfo(executeSQL, mentionedtable)[:result] - context[:mentionedTableInfo] = mentionedTableInfo - - initialstate = Dict{Symbol, Any}( - :reward=> 0, - :isterminal=> false, - :evaluation=> nothing, - :errormsg=> nothing, - :errorexplain=> nothing, - - :question=> query, - :code=> nothing, - :response=> nothing, - ) - - transitionargs = ( - # decisionMaker=getdata_decisionMaker, - # evaluator=getdata_evaluator, - # reflector=getdata_reflector, - context=context, - executeSQL=executeSQL, - text2textInstructLLM=text2textInstructLLM - ) - result_1, result_2 = LLMMCTS.runMCTS(initialstate, getdata_transition, transitionargs; - totalsample=1, maxdepth=3, maxiterations=1, explorationweight=1.0) - - if result_2[:isterminal] == true - return (result=result_2[:response], errormsg=nothing, success=true) # succues=true to finish getdata() + response = SQLexecution(executeSQL, query) + if response[:success] + extracted = extractContent_dataframe(response[:result], context, text2textInstructLLM) + response_ = (result=extracted, errormsg=nothing, success=true) + return response_ else - # return (response="Failed to act with the following error message: $(result_2[:errorexplain])", select=nothing, reward=0, success=false) - return (result="Failed to get the data. $(result_1[:errormsg])", - errormsg=result_1[:errormsg], success=false) + response_ = (result=nothing, errormsg=response[:errormsg], success=false) + return response_ end end +# function getdata(query::T, context::Union{Dict, Nothing}, executeSQL::Function, +# text2textInstructLLM::Function; +# )::NamedTuple{(:result, :errormsg, :success), Tuple{String, Union{String, Nothing}, Bool}} where {T<:AbstractString} + +# # get table info here because it'll be called only 1-time. If this function is in +# # getdata_decisionMaker(), it'll be called everytime +# mentionedtable = getTableNameFromSQL(query, text2textInstructLLM) +# mentionedTableInfo = tableinfo(executeSQL, mentionedtable)[:result] +# context[:mentionedTableInfo] = mentionedTableInfo + +# initialstate = Dict{Symbol, Any}( +# :reward=> 0, +# :isterminal=> false, +# :evaluation=> nothing, +# :errormsg=> nothing, +# :errorexplain=> nothing, + +# :question=> query, +# :code=> nothing, +# :response=> nothing, +# ) + +# transitionargs = ( +# # decisionMaker=getdata_decisionMaker, +# # evaluator=getdata_evaluator, +# # reflector=getdata_reflector, +# context=context, +# executeSQL=executeSQL, +# text2textInstructLLM=text2textInstructLLM +# ) +# result_1, result_2 = LLMMCTS.runMCTS(initialstate, getdata_transition, transitionargs; +# totalsample=1, maxdepth=3, maxiterations=1, explorationweight=1.0) + +# if result_2[:isterminal] == true +# return (result=result_2[:response], errormsg=nothing, success=true) # succues=true to finish getdata() +# else +# # return (response="Failed to act with the following error message: $(result_2[:errorexplain])", select=nothing, reward=0, success=false) +# return (result="Failed to get the data. $(result_1[:errormsg])", +# errormsg=result_1[:errormsg], success=false) +# end +# end """ @@ -290,7 +302,7 @@ julia> # Signature """ function getdata_evaluator(newstate, config) - + return (evaluation="None", score=0) end @@ -309,9 +321,9 @@ end # Signature """ function getdata_transition(state::T, args::NamedTuple - )::NamedTuple{(:newNodeKey, :newstate, :progressvalue), Tuple{String, T, Integer}} where {T<:AbstractDict} +)::NamedTuple{(:newNodeKey, :newstate, :progressvalue),Tuple{String,T,Integer}} where {T<:AbstractDict} + - # decisionMaker::Function = args[:decisionMaker] # evaluator::Function = args[:evaluator] # reflector::Function = args[:reflector] @@ -320,26 +332,26 @@ function getdata_transition(state::T, args::NamedTuple text2textInstructLLM::Function = args[:text2textInstructLLM] thought, sql = - if state[:code] !== nothing - result = getdata_decisionMaker(state, context, text2textInstructLLM) - result[:thought], result[:code] - else - nothing, state[:question] - end + if state[:code] !== nothing + result = getdata_decisionMaker(state, context, text2textInstructLLM) + result[:thought], result[:code] + else + nothing, state[:question] + end # make new state newNodeKey = GeneralUtils.uuid4snakecase() newstate = deepcopy(state) - + response, success, errormsg, reward, isterminal = if sql !== nothing response, success, errormsg, reward, isterminal = SQLexecution(executeSQL, sql) else - (result= nothing, - success= false, - errormsg= "SQL execution failed. An unexpected error occurred. Please try again.", - reward=0, - isterminal=false) + (result=nothing, + success=false, + errormsg="SQL execution failed. An unexpected error occurred. Please try again.", + reward=0, + isterminal=false) end println("getdata_transition() 1 ", @__FILE__, " ", @__LINE__) newstate[:code] = sql @@ -376,10 +388,10 @@ end # Signature """ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM::Function - )::NamedTuple{(:thought, :code, :success, :errormsg), Tuple{Union{String, Nothing}, Union{String, Nothing}, Bool, Union{String, Nothing}}} +)::NamedTuple{(:thought, :code, :success, :errormsg),Tuple{Union{String,Nothing},Union{String,Nothing},Bool,Union{String,Nothing}}} Hints = "None" - + # """ # Here are some useful SQL programs: # $usefulSQL @@ -414,78 +426,75 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM: # 2) ... # ... # code: ... - + # Let's begin! # """ - systemmsg = - """ - You are an assistant helping the user to execute SQL code from the user's query. + systemmsg = """ + You are an assistant helping the user to execute SQL code from the user's query. - At each round of conversation, the user will give you: - Context: ... - User intention: ... - Code executed from the last round: ... - Execution error: execution error of the last round code. + At each round of conversation, the user will give you: + Context: ... + User intention: ... + Code executed from the last round: ... + Execution error: execution error of the last round code. - You should consider the following guidelines: - - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search. + You should consider the following guidelines: + - Text information in the database is sometimes stored in lower case. If your search returns empty, try using lower case to search. - You should then respond to the user with: - 1) Understanding: - - State your understanding about the current situation. - 2) Reasoning: - - State your step by step reasoning about the current situation. - 3) Plan: Step-by-step instructions of how to complete the task. - - Focus on improving the code from the last round. - - Do not create any table in the database. - 4) Code: - - Write new improved code. - - Do not wrap the code and no comment as it will be executed directly without any modification against the database. + You should then respond to the user with: + 1) Understanding: + - State your understanding about the current situation. + 2) Reasoning: + - State your step by step reasoning about the current situation. + 3) Plan: Step-by-step instructions of how to complete the task. + - Focus on improving the code from the last round. + - Do not create any table in the database. + 4) Code: + - Write new improved code. + - Do not wrap the code and no comment as it will be executed directly without any modification against the database. - You should only respond in format as described below and nothing more: - Understanding: ... - Reasoning: ... - Plan: - 1) ... - 2) ... - ... - Code: ... - - Let's begin! - """ + You should only respond in format as described below and nothing more: + Understanding: ... + Reasoning: ... + Plan: + 1) ... + 2) ... + ... + Code: ... + + Let's begin! + """ noise = "" note_flag = "" for attempt in 1:10 - usermsg = - """ - Context: - $(context[:mentionedTableInfo]) - User intention: $(context[:userintention]) - Code executed from the last round: $(state[:code]) - Execution error: $(state[:errormsg]) - $noise - $note_flag - """ - - _prompt = - [ - Dict(:name=> "system", :text=> systemmsg), - Dict(:name=> "user", :text=> usermsg) - ] + usermsg = """ + Context: + $(context[:mentionedTableInfo]) + User intention: $(context[:userintention]) + Code executed from the last round: $(state[:code]) + Execution error: $(state[:errormsg]) + $noise + $note_flag + """ + + _prompt = + [ + Dict(:name => "system", :text => systemmsg), + Dict(:name => "user", :text => usermsg) + ] # put in model format prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct") - prompt *= - """ - <|start_header_id|>assistant<|end_header_id|> - """ + prompt *= """ + <|start_header_id|>assistant<|end_header_id|> + """ try response = text2textInstructLLM(prompt) responsedict = GeneralUtils.textToDict(response, - ["Understanding", "Reasoning", "Plan", "Code"]; - rightmarker=":", symbolkey=true, lowercasekey=true) + ["Understanding", "Reasoning", "Plan", "Code"]; + rightmarker=":", symbolkey=true, lowercasekey=true) _code = responsedict[:code] code = strip(_code) @@ -509,7 +518,7 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM: else end - println("--> getdata_decisionMaker() ", @__FILE__, " ", @__LINE__) + println("~~~ getdata_decisionMaker() ", @__FILE__, " ", @__LINE__) pprintln(Dict(responsedict)) return (thought=responsedict[:reasoning], code=code, success=true, errormsg=nothing) catch e @@ -523,7 +532,7 @@ function getdata_decisionMaker(state::Dict, context::Dict, text2textInstructLLM: end end return (thought=nothing, code=nothing, success=false, - errormsg="Failed to generate SQL after numerous attempts.") + errormsg="Failed to generate SQL after numerous attempts.") end """ Execute a given SQL. @@ -553,7 +562,7 @@ julia> response = SQLLLM.SQLexecution(executeSQL, sql) """ # function SQLexecution(executeSQL::Function, sql::T # )::NamedTuple{(:result, :success, :errormsg, :reward, :isterminal), Tuple{Union{DataFrame, Nothing}, Bool, Union{String, Nothing}, Integer, Bool}} where {T<:AbstractString} -# println("--> 1-01 ", @__FILE__, " ", @__LINE__) +# println("~~~ 1-01 ", @__FILE__, " ", @__LINE__) # #XXX dummy SQL. use for testing # # sql = "SELECT w.wine_name FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.\"food_name\" = 'lamb';" # # sql = " SELECT w.wine_name FROM wine w JOIN food f ON f.food_name = 'lamb' JOIN wine_food wf ON w.wine_id = wf.wine_id AND f.food_id = wf.food_id GROUP BY w.wine_name ORDER BY COUNT(DISTINCT w.wine_id) DESC;" @@ -567,37 +576,37 @@ julia> response = SQLLLM.SQLexecution(executeSQL, sql) # # add LIMIT to the SQL to prevent loading large data # sql = strip(sql) -# println("--> SQL 1", @__FILE__, " ", @__LINE__) +# println("~~~ SQL 1", @__FILE__, " ", @__LINE__) # println(sql) -# println("--> 1-02 ", @__FILE__, " ", @__LINE__) +# println("~~~ 1-02 ", @__FILE__, " ", @__LINE__) # if sql[end] != ';' # errorMsg = "Error, SQL execution failed because it does not ended with ';'" # return (result=nothing, success=false, errormsg=errorMsg, reward=0, isterminal=false) # end -# println("--> 1-03 ", @__FILE__, " ", @__LINE__) +# println("~~~ 1-03 ", @__FILE__, " ", @__LINE__) # if !occursin("LIMIT", sql) # # sql = sql[1:end-1] * " LIMIT 100;" # sql = sql[1:end-1] * " ORDER BY RANDOM() LIMIT 2;" # end -# println("--> SQL 2", @__FILE__, " ", @__LINE__) +# println("~~~ SQL 2", @__FILE__, " ", @__LINE__) # println(sql) -# println("--> 1-1 ", @__FILE__, " ", @__LINE__) +# println("~~~ 1-1 ", @__FILE__, " ", @__LINE__) # result = executeSQL(sql) -# println("--> 1-2 ", @__FILE__, " ", @__LINE__) +# println("~~~ 1-2 ", @__FILE__, " ", @__LINE__) # df = DataFrame(result) -# println("--> raw df ", df) +# println("~~~ raw df ", df) # tablesize = size(df) -# println("--> df size ", tablesize) -# println("--> 6 ", @__FILE__, " ", @__LINE__) +# println("~~~ df size ", tablesize) +# println("~~~ 6 ", @__FILE__, " ", @__LINE__) # row = tablesize[1] -# println("--> 7 ", @__FILE__, " ", @__LINE__) +# println("~~~ 7 ", @__FILE__, " ", @__LINE__) # if row == 0 # if 0 row # errorMsg = "The resulting table has 0 row. Possible causes: 1) SQL is incorrect 2) There is no data that match your search criteria." # return (result=nothing, success=false, errormsg=errorMsg, reward=0, isterminal=false) # end -# println("--> 8 ", @__FILE__, " ", @__LINE__) +# println("~~~ 8 ", @__FILE__, " ", @__LINE__) # df1 = # if row > 2 # # ramdom row to pick @@ -606,13 +615,13 @@ julia> response = SQLLLM.SQLexecution(executeSQL, sql) # df # end -# println("--> SQLexecution result ", @__FILE__, " ", @__LINE__) +# println("~~~ SQLexecution result ", @__FILE__, " ", @__LINE__) # println(df1) # return (result=df1, success=true, errormsg=nothing, reward=1, isterminal=true) # end function SQLexecution(executeSQL::Function, sql::T - )::NamedTuple{(:result, :success, :errormsg, :reward, :isterminal), Tuple{Union{DataFrame, Nothing}, Bool, Union{String, Nothing}, Integer, Bool}} where {T<:AbstractString} - +) where {T<:AbstractString} + try #XXX dummy SQL. use for testing # sql = "SELECT w.wine_name FROM wine w JOIN wine_food wf ON w.wine_id = wf.wine_id JOIN food f ON wf.food_id = f.food_id WHERE f.\"food_name\" = 'lamb';" @@ -635,7 +644,7 @@ function SQLexecution(executeSQL::Function, sql::T else error("Error, SQL execution failed because it does not ended with ';'") end - println("--> SQL ", @__FILE__, " ", @__LINE__) + println("~~~ SQL ", @__FILE__, " ", @__LINE__) println(sql) result = executeSQL(sql) @@ -649,24 +658,28 @@ function SQLexecution(executeSQL::Function, sql::T error("SQL execution failed. An unexpected error occurred. Please try again.") end - df1 = - if row > 2 - # ramdom row to pick - df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df - else - df - end + df1 = + if row > 2 + # ramdom row to pick + df[sample(1:nrow(df), 2, replace=false), :] # random select 2 rows from df + else + df + end - println("--> SQLexecution() ", @__FILE__, " ", @__LINE__) + println("~~~ SQLexecution() ", @__FILE__, " ", @__LINE__) println(df1) - return (result=df1, success=true, errormsg=nothing, reward=1, isterminal=true) + return (result=df1, success=true, errormsg=nothing) catch e + println("~~~ Error SQLexecution() 2 ", @__FILE__, " ", @__LINE__) io = IOBuffer() showerror(io, e) errorMsg = String(take!(io)) st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) println(errorMsg) - return (result=nothing, success=false, errormsg=errorMsg, reward=0, isterminal=false) + println("~~~ Error SQLexecution() 2.1 ", @__FILE__, " ", @__LINE__) + response = (result=nothing, success=false, errormsg=errorMsg) + println("~~~ Error SQLexecution() 2.2 ", @__FILE__, " ", @__LINE__) + return response end end @@ -687,100 +700,90 @@ end # Signature """ function extractContent_dataframe(df::DataFrame, context::Dict, text2textInstructLLM::Function - )::String - +)::String tablesize = size(df) row = tablesize[1] column = tablesize[2] - #[PENDING] Since selected column depend on the question, there should be a better way to select column on the fly, not hard coded like this. - df1 = - if column > 10 # assuming if columns > 10, agent is getting wine info but the info is too much - selectedcolumn = ["wine_id", - "wine_name", - "brand", - "manufacturer", - "region", - "country", - "wine_type", - "grape_variety", - "serving_temperature", - "intensity", - "sweetness", - "tannin", - "acidity", - "fizziness", - "tasting_notes"] - df1 = df[:, selectedcolumn] - else - df - end + # df1 = + # if column > 10 # assuming if columns > 10, agent is getting wine info but the info is too much + # selectedcolumn = ["wine_id", + # "wine_name", + # "winery", + # "region", + # "country", + # "wine_type", + # "grape", + # "serving_temperature", + # "intensity", + # "sweetness", + # "tannin", + # "acidity", + # "fizziness", + # "tasting_notes"] + # df1 = df[:, selectedcolumn] + # else + # df + # end + + df1 = df dfstr = dfToString(df1) - # println("--> df string") - # println(dfstr) - systemmsg = - """ - You are an assistant that readouts the resulting table after the user executing SQL command. + systemmsg = """ + You are an assistant that readouts the resulting table after the user executing SQL command. - At each round of conversation, the user will give you: - - User intention: ... - - Resulting table dimension: ... - - Resulting table: The resulting table after executing the user's intention. + At each round of conversation, the user will give you: + - User intention: ... + - Resulting table dimension: ... + - Resulting table: The resulting table after executing the user's intention. - You should then respond to the user with: - - About_resulting_table: - 1) What is the resulting table represent? - - Search_summary: - 1) Summarize the table's content based on the user intension in verbal English. - Here are some example: - Bad example (you are not Summarize the table content): there are 2 columns in the table i.e. "cash" and "number". - 2) Do not generate additional text. + You should then respond to the user with: + - About_resulting_table: + 1) What is the resulting table represent? + - Search_summary: + 1) Summarize the table's content based on the user intension in verbal English. + Here are some example: + Bad example (you are not Summarize the table content): there are 2 columns in the table i.e. "cash" and "number". + 2) Do not generate additional text. - You should only respond in format as described below: - About_resulting_table: ... - Search_summary: ... - - Let's begin! - """ + You should only respond in format as described below: + About_resulting_table: ... + Search_summary: ... - usermsg = - """ - User intention: $(context[:userintention]) - Resulting table: $dfstr - """ - - _prompt = - [ - Dict(:name=> "system", :text=> systemmsg), - Dict(:name=> "user", :text=> usermsg) - ] + Let's begin! + """ + usermsg = """ + User intention: $(context[:userintention]) + Resulting table: $dfstr + """ + _prompt = + [ + Dict(:name => "system", :text => systemmsg), + Dict(:name => "user", :text => usermsg) + ] # put in model format prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct") - prompt *= - """ - <|start_header_id|>assistant<|end_header_id|> - """ - + prompt *= """ + <|start_header_id|>assistant<|end_header_id|> + """ for i in 1:5 response = text2textInstructLLM(prompt) - responsedict = GeneralUtils.textToDict(response, ["About_resulting_table", "Search_summary"], - rightmarker=":", symbolkey=true) + responsedict = GeneralUtils.textToDict(response, ["About_resulting_table", "Search_summary"], + rightmarker=":", symbolkey=true) # result = dfstr - result = - """ - Summary: $(responsedict[:Search_summary]) - More details: $dfstr - """ + result = """ + Summary: $(responsedict[:Search_summary]) + More details: $dfstr + """ if row > 2 result *= "There are many more rows, but they are truncated because there are too many of them." end - println("--> extractContent_dataframe() ", @__FILE__, " ", @__LINE__) + println("~~~ extractContent_dataframe() ", @__FILE__, " ", @__LINE__) println(result) return result @@ -856,47 +859,44 @@ julia> result = SQLLLM.getTableNameFromSQL(sql, text2textInstructLLM) # Signature """ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function)::Vector{String} where {T<:AbstractString} - systemmsg = - """ - Extract table name out of the user query. + systemmsg = """ + Extract table name out of the user query. - At each round of conversation, the user will give you: - Query: ... + At each round of conversation, the user will give you: + Query: ... - You should then respond to the user with: - - table_name: a list of table name that the user mentioned in the query. - For example, ["color", "type"] + You should then respond to the user with: + - table_name: a list of table name that the user mentioned in the query. + For example, ["color", "type"] - You must only respond in format as described below: - table_name: ["...", "...", ...] - - Let's begin! - """ + You must only respond in format as described below: + table_name: ["...", "...", ...] - usermsg = - """ - Query: $sql - """ - - _prompt = - [ - Dict(:name=> "system", :text=> systemmsg), - Dict(:name=> "user", :text=> usermsg) - ] + Let's begin! + """ + + usermsg = """ + Query: $sql + """ + + _prompt = + [ + Dict(:name => "system", :text => systemmsg), + Dict(:name => "user", :text => usermsg) + ] # put in model format prompt = GeneralUtils.formatLLMtext(_prompt; formatname="llama3instruct") - prompt *= - """ - <|start_header_id|>assistant<|end_header_id|> - """ + prompt *= """ + <|start_header_id|>assistant<|end_header_id|> + """ for attempt in 1:5 try response = text2textInstructLLM(prompt) responsedict = GeneralUtils.textToDict(response, - ["table_name"], - rightmarker=":", symbolkey=true) + ["table_name"], + rightmarker=":", symbolkey=true) response = copy(JSON3.read(responsedict[:table_name])) return response diff --git a/test/runtest.jl b/test/runtest.jl index 17e569b..8c6af4d 100644 --- a/test/runtest.jl +++ b/test/runtest.jl @@ -5,44 +5,44 @@ using GeneralUtils, SQLLLM config = copy(JSON3.read("config.json")) -function executeSQL(sql) - DBconnection = LibPQ.Connection("host=192.168.88.12 port=5432 dbname=yiem_wine_assistant user=yiem password=yiem@Postgres_0.0") +function executeSQL(sql::T) where {T<:AbstractString} + DBconnection = LibPQ.Connection("host=192.168.88.12 port=10201 dbname=wineDB user=yiemtechnologies password=yiemtechnologies@Postgres_0.0") result = LibPQ.execute(DBconnection, sql) close(DBconnection) return result end -function text2textInstructLLM(prompt::String; max_tokens=2048) +function text2textInstructLLM(prompt::String) msgMeta = GeneralUtils.generate_msgMeta( config[:externalservice][:text2textinstruct][:mqtttopic]; - msgPurpose= "inference", - senderName= "yiemagent", - senderId= string(uuid4()), - receiverName= "text2textinstruct", - mqttBrokerAddress= config[:mqttServerInfo][:broker], - mqttBrokerPort= config[:mqttServerInfo][:port], + msgPurpose="inference", + senderName="yiemagent", + senderId=string(uuid4()), + receiverName="text2textinstruct", + mqttBrokerAddress=config[:mqttServerInfo][:broker], + mqttBrokerPort=config[:mqttServerInfo][:port], ) outgoingMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict( - :text=> prompt, - :kwargs=> Dict( - :max_tokens=> max_tokens, - :stop=> ["<|eot_id|>"], - :temperature=> 0.2, - ) + :msgMeta => msgMeta, + :payload => Dict( + :text => prompt, + :kwargs => Dict( + :num_ctx => 20480, + :temperature => 0.2, ) ) + ) - response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) - result = response[:response][:text] + _response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120) + response = _response[:response][:text] - return result + return response end + function executeSQLVectorDB(sql) - DBconnection = LibPQ.Connection("host=192.168.88.12 port=5433 dbname=SQLVectorDB user=yiemtechnologies@gmail.com password=yiem@Postgres_0.0") + DBconnection = LibPQ.Connection("host=192.168.88.12 port=10203 dbname=SQLVectorDB user=yiemtechnologies password=yiemtechnologies@Postgres_0.0") result = LibPQ.execute(DBconnection, sql) close(DBconnection) return result @@ -147,11 +147,14 @@ function querySQLVectorDB(state) return nothing end + + + # query = Dict(:text=> "How many wines from France do you have that can be paired with lamb?") # query = "How many wines are from United States?" -# query = "wine_type: red, country: France, sweetness: 1-2, intensity: 4-5" +query = "retailer: Yiem, wine_type: red, sweetness: 1-2, intensity: 4-5, wine price: 20-40" # query = "wine_type: white, country: United States, sweetness: 1-2, tannin: 3, food to be served with wine: pizza" -query = "wine_type: white, country: Austria, food to be served with wine: pork" +# query = "wine_type: white, country: Austria, food to be served with wine: pork" # query = "wine price: less than 25, wine_type: rose, country: France, sweetness: 2, tannin: 3, food to be served with wine: pizza" # query = Dict(:text=> "wine_type: white, country: France, sweetness: 1") result = SQLLLM.query(query, executeSQL, text2textInstructLLM;