From c8684dea318518928785d8b8d63403ad57683a14 Mon Sep 17 00:00:00 2001 From: tonaerospace Date: Fri, 13 Sep 2024 15:07:19 +0700 Subject: [PATCH] update --- .vscode/launch.json | 17 + .vscode/settings.json | 1 + Manifest.toml | 578 ++++++++++ Project.toml | 16 + codesnippet/documentTemplate.jl | 19 + codesnippet/executeUserScript.jl | 99 ++ codesnippet/showstacktrace.jl | 11 + previousVersion/0.0.1/Manifest.toml | 911 ++++++++++++++++ previousVersion/0.0.1/Project.toml | 11 + previousVersion/0.0.1/src/GeneralUtils.jl | 81 ++ previousVersion/0.0.1/src/interface.jl | 813 ++++++++++++++ previousVersion/0.0.1/test/etc.jl | 0 previousVersion/0.0.2/Manifest.toml | 911 ++++++++++++++++ previousVersion/0.0.2/Project.toml | 11 + previousVersion/0.0.2/src/GeneralUtils.jl | 81 ++ previousVersion/0.0.2/src/interface.jl | 841 +++++++++++++++ previousVersion/0.0.2/test/etc.jl | 0 previousVersion/0.0.3/Manifest.toml | 911 ++++++++++++++++ previousVersion/0.0.3/Project.toml | 12 + previousVersion/0.0.3/src/GeneralUtils.jl | 81 ++ previousVersion/0.0.3/src/interface.jl | 1108 +++++++++++++++++++ previousVersion/0.0.3/test/etc.jl | 0 src/GeneralUtils.jl | 89 ++ src/communication.jl | 640 +++++++++++ src/interface.jl | 1186 +++++++++++++++++++++ src/llmUtil.jl | 261 +++++ src/util.jl | 387 +++++++ test/runtest.jl | 67 ++ 28 files changed, 9143 insertions(+) create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 Manifest.toml create mode 100644 Project.toml create mode 100644 codesnippet/documentTemplate.jl create mode 100644 codesnippet/executeUserScript.jl create mode 100644 codesnippet/showstacktrace.jl create mode 100644 previousVersion/0.0.1/Manifest.toml create mode 100644 previousVersion/0.0.1/Project.toml create mode 100644 previousVersion/0.0.1/src/GeneralUtils.jl create mode 100644 previousVersion/0.0.1/src/interface.jl create mode 100644 previousVersion/0.0.1/test/etc.jl create mode 100644 previousVersion/0.0.2/Manifest.toml create mode 100644 previousVersion/0.0.2/Project.toml create mode 100644 previousVersion/0.0.2/src/GeneralUtils.jl create mode 100644 previousVersion/0.0.2/src/interface.jl create mode 100644 previousVersion/0.0.2/test/etc.jl create mode 100644 previousVersion/0.0.3/Manifest.toml create mode 100644 previousVersion/0.0.3/Project.toml create mode 100644 previousVersion/0.0.3/src/GeneralUtils.jl create mode 100644 previousVersion/0.0.3/src/interface.jl create mode 100644 previousVersion/0.0.3/test/etc.jl create mode 100644 src/GeneralUtils.jl create mode 100644 src/communication.jl create mode 100644 src/interface.jl create mode 100644 src/llmUtil.jl create mode 100644 src/util.jl create mode 100644 test/runtest.jl diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..7f264b6 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "julia", + "request": "launch", + "name": "Run active Julia file", + "program": "${file}", + "stopOnEntry": false, + "cwd": "${workspaceFolder}", + "juliaEnv": "${command:activeJuliaEnvironment}" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..615d5c2 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,578 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.4" +manifest_format = "2.0" +project_hash = "b6f51053f1fe5e6c447c7d94689166d2519b40e2" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "6c834533dc1fabd820c1db03c839bf97e45a3fab" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.14" + +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "7eee164f122511d3e4e1ebadb7956939ea7e1c77" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "1.3.6" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.6" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "04c738083f29f86e62c8afc341f0967d8717bdb8" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.6.1" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Distributions]] +deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "e6c693a0e4394f8fda0e51a5bdf5aef26f8235e9" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.111" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + DistributionsTestExt = "Test" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.22" + + [deps.FilePathsBase.extensions] + FilePathsBaseMmapExt = "Mmap" + FilePathsBaseTestExt = "Test" + + [deps.FilePathsBase.weakdeps] + Mmap = "a63ad114-7e13-5084-954f-fe012c677804" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "fd0002c0b5362d7eb952450ad5eb742443340d6e" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.12.0" +weakdeps = ["PDMats", "SparseArrays", "Statistics"] + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStatisticsExt = "Statistics" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.HypergeometricFunctions]] +deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "7c4195be1649ae622304031ed46a2f4df989f1eb" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.24" + +[[deps.InlineStrings]] +git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.2" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.InvertedIndices]] +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.0" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.5.0" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "eb3edce0ed4fa32f75a0a11217433c31d56bd48b" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.14.0" + + [deps.JSON3.extensions] + JSON3ArrowExt = ["ArrowTypes"] + + [deps.JSON3.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +git-tree-sha1 = "4b415b6cccb9ab61fec78a621572c82ac7fa5776" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.9.35" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.1" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.4.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "a2d09619db4e765091ee5c6ffe8872849de0feea" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.28" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.LoweredCodeUtils]] +deps = ["JuliaInterpreter"] +git-tree-sha1 = "1ce1834f9644a8f7c011eb0592b7fd6c42c90653" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +version = "3.0.1" + +[[deps.MQTTClient]] +deps = ["Distributed", "Random", "Sockets"] +git-tree-sha1 = "f2597b290d4bf17b577346153cd2ddf9accb5c26" +uuid = "985f35cc-2c3d-4943-b8c1-f0931d5f0959" +version = "0.3.1" +weakdeps = ["PrecompileTools"] + + [deps.MQTTClient.extensions] + PrecompileMQTT = "PrecompileTools" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+1" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.1.10" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.23+4" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+2" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "949347156c25054de2db3b166c52ac4728cbad65" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.31" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.10.0" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "66b20dd35966a748321d3b2537c4584cf40387c7" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.3.2" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.PtrArrays]] +git-tree-sha1 = "f011fbb92c4d401059b2212c05c0601b70f8b759" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.2.0" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "e237232771fdafbae3db5c31275303e056afaa9f" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.10.1" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Revise]] +deps = ["CodeTracking", "Distributed", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] +git-tree-sha1 = "7b7850bb94f75762d567834d7e9802fc22d62f9c" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.5.18" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e60724fd3beea548353984dc61c943ecddb0e29a" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.3+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "ff11acffdb082493657550959d4feb4b6149e73a" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.5" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.10.0" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "2f5d4697f21388cbe1ff299430dd169ef97d7e14" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.4.0" + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + + [deps.SpecialFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.10.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.7.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "5cf7606d6cef84b543b483848d4ae08ad9832b21" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.3" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "cef0472124fab0695b58ca35a77c6fb942fdab8a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.1" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.3.4" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.10.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.2.1+1" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "e84b3a11b9bece70d14cce63406bbc79ed3464d2" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.2" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.52.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..2f29082 --- /dev/null +++ b/Project.toml @@ -0,0 +1,16 @@ +name = "GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +authors = ["tonaerospace "] +version = "0.1.0" + +[deps] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +MQTTClient = "985f35cc-2c3d-4943-b8c1-f0931d5f0959" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/codesnippet/documentTemplate.jl b/codesnippet/documentTemplate.jl new file mode 100644 index 0000000..623b4c9 --- /dev/null +++ b/codesnippet/documentTemplate.jl @@ -0,0 +1,19 @@ +""" + +# Arguments + - `v::Integer` + dummy variable + +# Return + +# Example +```jldoctest +julia> +``` + +# TODO + - [] update docstring + - [] implement the function + +# Signature +""" \ No newline at end of file diff --git a/codesnippet/executeUserScript.jl b/codesnippet/executeUserScript.jl new file mode 100644 index 0000000..3e0ebc4 --- /dev/null +++ b/codesnippet/executeUserScript.jl @@ -0,0 +1,99 @@ +""" prompt +I want to have my main function loop running and execute a user-defined script: usermodule_1.jl, + usermodule _2.jl and so on. +1). Write a main.jl with main function loop to execute user-defined scripts sequentially. +2). Store each script result in respective variable. +3). make sure no global variables contamination Give me a complete example. +""" + +# ---------------------------------------------- 100 --------------------------------------------- # + +"""To achieve your goal of running user-defined scripts sequentially within a main function loop in Julia, storing each script's result in a respective variable, and ensuring no global variable contamination, you can follow the example below: +Create a main.jl script with a main function loop to execute user-defined scripts sequentially:""" + +# -------------------------------------------- main.jl ------------------------------------------- # +function main() + script_results = Dict{String, Any}() + + for i in 1:2 + script_name = "usermodule_$i.jl" + include(script_name) + + # Assuming each user-defined script has a function named 'run_script' that returns a result + result = run_script() + + script_results[script_name] = result + end + + return script_results +end + +results = main() + + +# ---------------------------------------- usermodule_1.jl --------------------------------------- # +module UserModule1 + +export run_script + +function run_script() + # Your script logic here + result = "Result from UserModule1" + return result +end + +end + +# ---------------------------------------- usermodule_2.jl --------------------------------------- # +module UserModule2 + +export run_script + +function run_script() + # Your script logic here + result = "Result from UserModule2" + return result +end + +end + +""" Ensure no global variable contamination by encapsulating each script's logic within a module and exporting only the necessary functions. + +When you run main.jl, it will sequentially execute usermodule_1.jl and usermodule_2.jl, store the results in a dictionary script_results, and return it. Each script's logic is encapsulated within its module to prevent global variable contamination. + +This approach allows you to run user-defined scripts sequentially, store their results, and maintain a clean and modular structure in your Julia code. +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/codesnippet/showstacktrace.jl b/codesnippet/showstacktrace.jl new file mode 100644 index 0000000..17df90d --- /dev/null +++ b/codesnippet/showstacktrace.jl @@ -0,0 +1,11 @@ +try + # put code here +catch e + io = IOBuffer() + showerror(io, e) + errorMsg = String(take!(io)) + st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) + @warn "Error occurred: $errorMsg\n$st" +end + + diff --git a/previousVersion/0.0.1/Manifest.toml b/previousVersion/0.0.1/Manifest.toml new file mode 100644 index 0000000..ebad44a --- /dev/null +++ b/previousVersion/0.0.1/Manifest.toml @@ -0,0 +1,911 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.1" +manifest_format = "2.0" +project_hash = "91349a0eb57aed703e8465e44731cca768229f09" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "8bc0aaec0ca548eb6cf5f0d7d16351650c1ee956" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.3.2" +weakdeps = ["ChainRulesCore"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.6.2" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgCheck]] +git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.3.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "dbf84058d0a8cbbadee18d25cf606934b22d7c66" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.4.2" + +[[deps.BangBang]] +deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] +git-tree-sha1 = "e28912ce94077686443433c2800104b061a827ed" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.3.39" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.CEnum]] +git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.2" + +[[deps.CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "442d989978ed3ff4e174c928ee879dc09d1ef693" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "4.3.2" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] +git-tree-sha1 = "498f45593f6ddc0adff64a9310bb6710e851781b" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "0.5.0+1" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "0.2.2" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "5248d9c45712e51e27ba9b30eebec65658c6ce29" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.6.0+0" + +[[deps.CUDNN_jll]] +deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "2918fbffb50e3b7a0b9127617587afa76d4276e8" +uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" +version = "8.8.1+0" + +[[deps.Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] +git-tree-sha1 = "1cdf290d4feec68824bfb84f4bfc9f3aba185647" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.51.1" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.16.0" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[deps.Compat]] +deps = ["UUIDs"] +git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.6.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.2+0" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + + [deps.CompositionsBase.weakdeps] + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.ConstructionBase]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "738fec4d684a9a6ee9598a8bfee305b26831f28c" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.2" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.ContextVariablesX]] +deps = ["Compat", "Logging", "UUIDs"] +git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" +uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" +version = "0.1.3" + +[[deps.DataAPI]] +git-tree-sha1 = "e8119c1a33d267e16108be441a287a6981ba1630" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.14.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "d1fff3a548102f48987a52a2e0d114fa97d730f0" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.13" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] +git-tree-sha1 = "eead66061583b6807652281c0fbf291d7a9dc497" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.90" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.DualNumbers]] +deps = ["Calculus", "NaNMath", "SpecialFunctions"] +git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" +uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" +version = "0.6.8" + +[[deps.ExprTools]] +git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.9" + +[[deps.FLoops]] +deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] +git-tree-sha1 = "ffb97765602e3cbe59a0589d237bf07f245a8576" +uuid = "cc61a311-1640-44b5-9fba-1b764f453329" +version = "0.2.1" + +[[deps.FLoopsBase]] +deps = ["ContextVariablesX"] +git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" +uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" +version = "0.1.1" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "fc86b4fd3eff76c3ce4f5e96e2fdfa6282722885" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.0.0" + +[[deps.Flux]] +deps = ["Adapt", "CUDA", "ChainRulesCore", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "NNlibCUDA", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote", "cuDNN"] +git-tree-sha1 = "3e2c3704c2173ab4b1935362384ca878b53d4c34" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.13.17" + + [deps.Flux.extensions] + AMDGPUExt = "AMDGPU" + FluxMetalExt = "Metal" + + [deps.Flux.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "00e252f4d706b3d55a8863432e742bf5717b498d" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.35" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Functors]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "478f8c3145bb91d82c2cf20433e8c1b30df454cc" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.4.4" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] +git-tree-sha1 = "2e57b4a4f9cc15e85a24d603256fe08e527f48d1" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "8.8.1" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.5" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "cb090aea21c6ca78d59672a7e7d13bd56d09de64" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.20.3" + +[[deps.HypergeometricFunctions]] +deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "84204eae2dd237500835990bcade263e27674a93" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.16" + +[[deps.IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "eac00994ce3229a464c2847e956d77a2c64ad3a5" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.10" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.4.1" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "SnoopPrecompile", "StructTypes", "UUIDs"] +git-tree-sha1 = "84b10656a41ef564c39d2d477d7236966d2b5683" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.12.0" + +[[deps.JuliaVariables]] +deps = ["MLStyle", "NameResolution"] +git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" +uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" +version = "0.2.4" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "b48617c5d764908b5fac493cd907cf33cc11eec1" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.6" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "5007c1421563108110bbd57f63d8ad4565808818" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "5.2.0" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "1222116d7313cdefecf3d45a2bc1a89c4e7c9217" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.22+0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "0a1b7c2863e44523180fdb3146534e265a91870b" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.23" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.MLStyle]] +git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" +uuid = "d8e11817-5142-5d16-987a-aa16d5891078" +version = "0.4.17" + +[[deps.MLUtils]] +deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] +git-tree-sha1 = "3504cdb8c2bc05bde4d4b09a81b01df88fcbbba0" +uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" +version = "0.4.3" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.10" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.MicroCollections]] +deps = ["BangBang", "InitialValues", "Setfield"] +git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.1.4" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.1.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] +git-tree-sha1 = "72240e3f5ca031937bd536182cb2c031da5f46dd" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.8.21" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + +[[deps.NNlibCUDA]] +deps = ["Adapt", "CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics", "cuDNN"] +git-tree-sha1 = "f94a9684394ff0d325cc12b06da7032d8be01aaf" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.2.7" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.0.2" + +[[deps.NameResolution]] +deps = ["PrettyPrint"] +git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" +uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" +version = "0.1.5" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OneHotArrays]] +deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] +git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c" +uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" +version = "0.2.4" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "6a01f65dd8583dee82eecc2a19b0ff21521aa749" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.2.18" + +[[deps.OrderedCollections]] +git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.0" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "67eae2738d63117a196f497d7db789821bce61d1" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.17" + +[[deps.Parsers]] +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.5.8" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.2" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.0" + +[[deps.PrettyPrint]] +git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" +uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" +version = "0.2.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.8.2" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.6.1" + +[[deps.RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.5.3" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.0+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.1" + +[[deps.ShowCases]] +git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" +uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" +version = "0.1.0" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.SnoopPrecompile]] +deps = ["Preferences"] +git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.3" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "a4ada03f999bd01b3a25dcaa30b2d929fe537e00" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.1.0" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "ef28127915f4229c971eb43f3fc075dd3fe91880" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.2.0" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] +git-tree-sha1 = "832afbae2a45b4ae7e831f86965469a24d1d8a83" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.5.26" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.6.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.0" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.0" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StructArrays]] +deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] +git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.15" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.10.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.10.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.23" + +[[deps.Transducers]] +deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] +git-tree-sha1 = "a66fb81baec325cf6ccafa243af573b031e87b00" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.77" + + [deps.Transducers.extensions] + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "ea37e6066bf194ab78f4e747f5245261f17a7175" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.1.2" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "5be3ddb88fc992a7d8ea96c3f10a49a7e98ebc7b" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.62" + + [deps.Zygote.extensions] + ZygoteColorsExt = "Colors" + ZygoteDistancesExt = "Distances" + ZygoteTrackerExt = "Tracker" + + [deps.Zygote.weakdeps] + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "977aed5d006b840e2e40c0b48984f7463109046d" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.3" + +[[deps.cuDNN]] +deps = ["CEnum", "CUDA", "CUDNN_jll"] +git-tree-sha1 = "f65490d187861d6222cb38bcbbff3fd949a7ec3e" +uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" +version = "1.0.4" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/previousVersion/0.0.1/Project.toml b/previousVersion/0.0.1/Project.toml new file mode 100644 index 0000000..d09189f --- /dev/null +++ b/previousVersion/0.0.1/Project.toml @@ -0,0 +1,11 @@ +name = "GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +authors = ["tonaerospace "] +version = "0.1.0" + +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" diff --git a/previousVersion/0.0.1/src/GeneralUtils.jl b/previousVersion/0.0.1/src/GeneralUtils.jl new file mode 100644 index 0000000..a62eb89 --- /dev/null +++ b/previousVersion/0.0.1/src/GeneralUtils.jl @@ -0,0 +1,81 @@ +module GeneralUtils + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue + + +include("interface.jl") +using .interface + + +#------------------------------------------------------------------------------------------------100 + +""" version 0.0.1 + Todo: + - [] + + Change from version: + - + + All features + +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # generalUtils \ No newline at end of file diff --git a/previousVersion/0.0.1/src/interface.jl b/previousVersion/0.0.1/src/interface.jl new file mode 100644 index 0000000..6bf67e1 --- /dev/null +++ b/previousVersion/0.0.1/src/interface.jl @@ -0,0 +1,813 @@ +module interface + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, replaceMoreThan, + replaceLessThan, cartesianAssign!, sumAlongDim3, matMul_3Dto3D_manyTo1batch, + matMul_3Dto4D_batchwise + +using JSON3, DataStructures, Distributions, Flux, CUDA + +#------------------------------------------------------------------------------------------------100 + +noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) +findNotZero(x::AbstractVector) = findall( (!iszero).(x) ) +replaceMoreThan(i, target) = i > target ? target : i +replaceLessThan(i, target) = i < target ? target : i +precision(x::Array{<:Array}) = ( std(mean.(x)) / mean(mean.(x)) ) * 100 +precision(x::Array) = std(x) / mean(x) * 100 +replaceAt!(x::AbstractVector, ind::Number, value::Number) = x[ind] = value +notZero(x::AbstractVector) = (!iszero).(x) +Zero(x::AbstractVector) = iszero.(x) +isNan(x::AbstractVector) = isnan.(x) +isInf(x::Number) = abs(x) === Inf +isInf(x::AbstractVector) = isinf.(x) +isLess(x::AbstractVector, target::Number) = isless.(x, target) +isMore(x::Number, target::Number) = x > target +isMore(x::AbstractVector, target::Number) = isMore.(x, target) +absolute(x::AbstractVector) = abs.(x) +vecEleMul(x::AbstractVector, y::AbstractVector) = x .* y +vecEleMul(x::Number, y::AbstractVector) = x .* y +expDecay(initialValue::Number, decayFactor::Number, timePass::Number) = + initialValue * (1 - decayFactor)^timePass +mul!(x::AbstractVector, y::AbstractVector) = x .*= y +mul(x::AbstractVector, y::AbstractVector) = x .* y + +ReLu(x::Number) = max(0, x) + +updateVector!(x::AbstractVector, target::Number) = x .= target +updateVector!(x::AbstractVector, target::AbstractArray) = x .= target + +function selectAdd!(x::AbstractVector, ind::AbstractVector, value::AbstractVector) + @. x = x + (ind * value) +end + + +""" findIndex(input::String, target::Char) + + Find target index inside a collection. + Return 1) Bool array of matched target + 2) CartesianIndex of every matched target +""" +function findIndex(input::String, target::Char) + match_position = [] + for i in input + if i == target + append!(match_position, 1) + else + append!(match_position, 0) + end + end + match_index = findall(isequal.(match_position, 1)) + + return match_position, match_index +end + +function findIndex(input::Array, target::Number) + match_position = isequal.(input, target) + match_index = findall(match_position) + return match_position, match_index +end + +# function findIndex(input::Array, target::Array) +# match_position = isone.(zeros(length(input))) +# for i in target +# match_position = match_position + isequal.(input, i) +# end + +# match_position = replaceMoreThan.(match_position, 1) +# match_index = findall(isone.(match_position)) # Findall donot work with Int64 vector [1, 0, 0, 1]. +# # It only works with BitVector. isone() converts Int64 vector [1, 0, 0, 1] into +# # BitVector [1, 0, 0, 1] + +# return match_position, match_index +# end + +function findIndex(input::Array, target::Symbol) + match_position = isequal.(input, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::String) + match_position = isequal.(collection, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::Array{String}) + match_position = nothing + match_index = nothing + + for i in target + match_pos = isequal.(collection, i) + match_ind = findall(match_pos) + + if match_position === nothing + match_position = match_pos + + else + match_position = hcat(match_position, match_pos) + end + + if match_index === nothing + match_index = match_ind + else + match_index = hcat(match_index, match_ind) + end + end + + return match_position, match_index +end + +function findIndex(collection::OrderedDict, target::Symbol) + collection_keys = keys(collection) + collection_keys_array = [i for i in collection_keys] + match_position = isequal.(collection_keys_array, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findMax(collection::AbstractVector) + maxValue, maxIndex = findmax(collection) + matchPosition = isequal.(collection, maxValue) + return maxValue, maxIndex, matchPosition +end + + + +""" read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + + with multiple text file in a folder, + this function read x_th text file in a folder (filename is sorted by OS) + + # Example + utils.read_textfile_by_index(cleaned_data_path, 2) + read 2nd txt file in a folder +""" +function read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + if isdir(folder_path) + filenumber = length(readdir(folder_path)) + + if read_file_number > filenumber + error("you specified read_file_number = $read_file_number which is out + of range, the cleaned data folder has only $filenumber files") + return nothing, nothing, nothing + else + content = 0 + # open each file in the directory and read + filename = readdir(folder_path, join=true, sort=false)[read_file_number] + f = open(filename) + content = readlines(f) + # content = read(f) + close(f) + end + + return read_file_number, filename, content + else + error("ERROR no file or folder at $folder_path") + return nothing, nothing, nothing + end + +end + +#------------------------------------------------------------------------------------------------100 + +""" Array_to_JSON3_str(data::AbstractArray) + + encode Array to JSON3 String + + # Example + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] + json3_str = Array_to_JSON3_str(a) + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} +""" +function Array_to_JSON3_str(data::AbstractArray) + d = Dict("Array"=> data, "size"=>size(data)) + json3_str = JSON3.write(d) + return json3_str +end + +#------------------------------------------------------------------------------------------------100 + +""" JSON3_str_to_Array(json3_str::String) + + decode JSON3 String to Array + + # Example + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} + a = JSON3_str_to_Array(json3_str) + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] +""" +function JSON3_str_to_Array(json3_str::String) + d = JSON3.read(json3_str) + array = reshape(Array(d.Array), (d.size[1], d.size[2])) + return array +end + +#------------------------------------------------------------------------------------------------100 + +""" Convert JSON3.read object to OrderedDict + + # Example + dict = dictionary(["a"=>4, "b"=>6]) + OrDict = OrderedDict(dict) + jsonString = JSON3.write(OrDict) # use jsonString to exchange. One can save it to file or send it thru pub/sub + jsonObject = JSON3.read(jsonString) + OrDict2 = JSON3read_to_OrDict(jsonObject) # example here + Adict2 = dictionary(OrDict2) + + Andyferris's github https://github.com/andyferris/Dictionaries.jl +""" +function JSON3read_to_OrDict(x) + dict = OrderedDict() + for (k, v) in x + k = string(k) + dict[k] = v + end + return dict +end + +#------------------------------------------------------------------------------------------------100 + +""" + print time of cpu executtion at the line inwhich this macro is used +""" +macro timeline(expr) + quote + print("line ", $(__source__.line), ": ") + @time $(esc(expr)) + end +end + + +batchindex(batch_counter::Number, batch_size::Number; offset=0) = + (offset + (batch_counter-1) * batch_size + 1) : offset + (batch_counter * batch_size) + +function flip_true_false(x::Bool) + if x == true + x = false + elseif x == false + x = true + else + error("undefined condition line $(@__LINE__)") + end + + return x +end + +function flip_true_false(x::Int) + if x == 1 + x = 0 + elseif x == 0 + x = 1 + else + throw("not define input of type $(typeof(x)) yet") + end + + return x +end + +""" + Return drawed index + # Example + drawed_index = randomWithProb([0.5, 0.2, 0.3]) + +""" +randomWithProb(probability::AbstractVector) = rand(Distributions.Categorical(probability)) # return drawed index + +""" + Draw from choices according to its probability. + Probability range is 0.0 to 1.0 and all probability must summed up to 1 + (may get probability from NNlib's softmax function) + + # Example + + draw = draw_choices([true, false, nothing], [0.5, 0.2, 0.3]) +""" +function randomChoiceWithProb(choices::Array, probability::Array) + if length(choices) != length(probability) + error("random is not possible, choices array length != probability array length") + elseif sum(probability) != 1.0 + error("probability does not sum to 1.0") + end + + return choices[randomWithProb(probability)] +end + +function randomChoiceOnTarget(target::Number, targetMatch::Number, choices::AbstractVector, + probability::AbstractVector) + if length(choices) != length(probability) + throw("random is not possible, choices array length != probability array length") + end + return target == targetMatch ? randomChoiceWithProb(choices, probability) : target + # dist = Distributions.Categorical(probability) + # draw_result = choices[rand(dist)] +end + +function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector, + probability::AbstractVector) + return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,)) +end + +function linearly_weighted_avg(a::Array) + total = 0.0 + for (i, v) in enumerate(a) + total = total + (i * v) + end + + return total / sum(a) +end + + +""" Convert String that is holded inside a variable to Symbol +# Example +x = "hello" # x is a variable holding String "hello" \n +y = variable_to_symbol(x) # y holds :hello +""" +function variable_str_to_symbol(variable) + semi = :($variable) + symbol = Symbol(semi) + + return symbol +end + + +""" get useable type of specified fieldname inside a composite struct + + # Example + julia> @Base.kwdef mutable struct some_struct + a::Union{Bool, Nothing} = nothing + b::Union{Float64, Nothing} = nothing + c::Union{Int64, AbstractFloat} = 3.5 + d::Union{String, Nothing} = nothing + end + + julia> a = some_struct() + julia> fieldname_useable_type(some_struct, :c) =result=> [Int64, Float64] +""" +function fieldname_useable_type(somestruct, fieldname::Symbol; + test_types=[2.0, 2, true, :h, "str", 'c', missing, nothing])::Vector{DataType} + new_instance = somestruct() + useable_type = [] + + for i in test_types + try + new_instance.:($fieldname) = i + type = typeof(new_instance.:($fieldname)) + if type ∉ useable_type + push!(useable_type, type) + end + catch + + end + end + + return useable_type +end + + +function randomNoRepeat(drawOptions::Array, draw_number::Integer; + exclude_list::Union{AbstractArray,Nothing}=nothing) + draw_option = copy(drawOptions) + draw_option = isnothing(exclude_list) ? draw_option : + filter!(x -> x ∉ exclude_list, draw_option) + shuffle!(draw_option) + drawed_items = [] + while length(drawed_items) < draw_number + push!(drawed_items, pop!(draw_option)) + end + return drawed_items +end + +""" using cron to schedule backup job by + 1. sudo nano /etc/crontab <<< this is a system-wide cron file + 2. to execute julia file @ 2.00am everyday add the following line at the buttom of the file + 0 2 * * * root julia-1.7 /home/syncthing_backup_script.jl + + Requirements using Dates +""" +function folderBackup(sourceFolderAbsolutePath::String, # absolute path to folder to be backuped + backupFolderAbsolutePath::String; # absolute path to folder used to store backup file + totalBackupFiles::Integer=7, # total backup file, the oldest will be deleted + containerName::Union{Array{String}, Nothing}=nothing) # container using source_folder + + sep = (Sys.iswindows() ? "\\" : '/') + + if sourceFolderAbsolutePath[end] == sep + sourceFolderAbsolutePath = sourceFolderAbsolutePath[1:end-1] + end + if backupFolderAbsolutePath[end] != sl + backupFolderAbsolutePath = backupFolderAbsolutePath * sep + end + + if isdir(backupFolderAbsolutePath) + else + mkpath(backupFolderAbsolutePath) + end + + # stop running docker container service + if containerName !== nothing + println("stop running services") + for i in containerName + try run(`docker stop $i`) catch; end + sleep(10) # wait for services to stop + end + end + + # do backup + println("doing backup now") + timestamp = string(Dates.now()) + name = split(sourceFolderAbsolutePath, sep)[end] * "--" + filename = name * timestamp * ".zip" # resulting compressed filename + run(`chmod -R a+rwx $sourceFolderAbsolutePath`) + # zip -r [destination+filename] [source folder to be zipped] + run(`zip -r $(backupFolderAbsolutePath * filename) $sourceFolderAbsolutePath`) + + # check if total backup file is more than user specified, if yes, delete the oldest backup + backupFiles = readdir(backupFolderAbsolutePath) + + while length(backupFiles) > totalBackupFiles + run(`rm $(backupFolderAbsolutePath * backupFiles[1])`) + backupFiles = readdir(backupFolderAbsolutePath) + end + + # start docker services + if containerName !== nothing + println("start services") + for i in containerName + try run(`docker start $i`) catch; end + sleep(10) # wait for services to stop + end + end +end + +function lowerclip!(data::AbstractVector, lowerbound::Number) + replace!(x -> x < lowerbound ? lowerbound : x, data) +end + +function upperclip!(data::AbstractVector, upperbound::Number) + replace!(x -> x > upperbound ? upperbound : x, data) +end + +function normalise(x::AbstractArray, mu, std) + ϵ = oftype(x[1], 1e-5) + μ = mu + # σ = std(x, dims=dims, mean=μ, corrected=false) # use this when Zygote#478 gets merged + σ = std + return (x .- μ) ./ (σ .+ ϵ) +end + +function minMaxScaler(x::AbstractVector) + min = findmin(x)[1] + max = findmax(x)[1] + + scaler(a::Number, min::Number, max::Number) = (a-min) / (max-min) + return scaler.(x, min, max) +end + +""" a = [-1e200, -1e-200, 1e200, 1e-200] \n + result = vtclamp.(a, 1e-6, 1e6, -1e6, -1e-6) +""" +function customclamp(x::Number, poslo::Number, poshi::Number, + neglo::Number, neghi::Number) + signx = sign(x) + if signx == -1 + if neghi < x < 0 + return neghi + elseif x < neglo + return neglo + else + return x + end + elseif signx == +1 + if poshi < x + return poshi + elseif 0 < x < poslo + return poslo + else + return x + end + end +end + +function unitVec(x::AbstractVector) + y = √(sum(x.^2)) + return x./y +end + +function replaceAt!(x::AbstractVector, ind::AbstractVector, value::Number) + for i in ind + x[i] = value + end +end + +function signbitVec(x::AbstractVector) + sign = signbit.(x) * 1 + signVec = replace(s -> s == 0 ? -1 : s, sign) + return signVec +end + +function deleteall!(x::AbstractVector) + for i in 1:length(x) + deleteat!(x, 1) + end +end + +""" Select specific range of vectors in a dict, return a new dict +# Example +dict = Dict(:a => [1:5...], + :b => [6:10...]) + +call -> selectRange(dict, 1:3) +return -> Dict{Any, Any} with 2 entries: + :a => [1, 2, 3] + :b => [6, 7, 8] +""" +function selectRange(d::Dict{Symbol, <:AbstractVector}, range) + newDict = Dict{Symbol, AbstractVector}() + for (k, v) in d + newDict[k] = v[range] + end + + return newDict +end + +""" Assign value to a given Dict by array of keys + + # Example + d = Dict( + :a1=> Dict(:c=> 5), + :a2=> Dict( + :k=> 10, + :b=> Dict( + :s=> "target", + ) + ) + ) + index = [:a2, :b, :s] \n + assignDict!(d, [:a2, :b, :s], "wow") + + return 1 if no target key in a given dict. +""" +function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign) + wd = nothing + for i in accessArray + println(i) + if i != accessArray[end] + if wd === nothing && haskey(dict, i) + wd = Ref(dict[i]) + elseif wd.x !== nothing && haskey(wd.x, i) + wd = Ref(wd.x[i]) + else + return 1 # error, no target key in a given dict. + end + else + wd.x[i] = valueToAssign + return 0 + end + end +end + +""" convert hour(0-23), minute(0-59) into julia time object +# Example + time +""" +function iTime(h::Integer, m::Integer) + if h == 0 + h = 12 + ampm = "am" + elseif 1 <= h <= 11 + ampm = "am" + elseif h == 12 + ampm = "pm" + elseif 13 <= h <= 23 + h = h - 12 + ampm = "pm" + else + error("hour out of range") + end + + m = m < 10 ? "0$m" : m + t = "$h:$m$ampm" + + return Time(t, "HH:MMp") +end + +""" replace a number according to the limit + if value is lower than lowerbound return lowerbound replacement value + if value is more than upperbound return upperbound replacement value + + # Example + limitvalue(4, (-5 => 0), (5 => 5)) +""" +function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair) + lwLimit, lwReplace = lowerbound + upLimit, upReplace = upperbound + + if v < lwLimit + v = lwReplace + elseif v > upLimit + v = upReplace + else + end + return v +end + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all slice of 3D another matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> input = rand(32, 32, 128) # batch at 3rd dim + julia> weight = rand(32, 32, 1024) + julia> r = matMul_3Dto3D_manyTo1batch(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul_3Dto3D_manyTo1batch(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if resultStorage === nothing + resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) + end + + c = [slice .* b for slice in eachslice(a, dims=3)] + resultStorage .= cat(c..., dims=4) + + return resultStorage +end +""" GPU version of batchMatEleMul + + Example + julia> using Flux, CUDA + julia> device = Flux.CUDA.functional() ? gpu : cpu + julia> if device == gpu CUDA.device!(0) end + julia> input = rand(32, 32, 128) |> gpu # 128-batches + julia> weight = rand(32, 32, 1024) |> gpu # 1-batch + julia> r = batchMatEleMul_gpu(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul_3Dto3D_manyTo1batch(a::CuArray, b::CuArray; + resultStorage::Union{CuArray, Nothing}=nothing, threads=128) + asize = [size(a)...] + bsize = [size(b)...] + if resultStorage === nothing + resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) |> gpu + end + CUDA.@sync begin + @cuda threads=threads matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) + end + return resultStorage +end +function matMul_3Dto3D_manyTo1batch_gpu!(a::CuArray, b::CuArray, resultStorage::CuArray) + _, _, _, p = size(resultStorage) + + index = threadIdx().x # this example only requires linear indexing, so just use `x` + stride = blockDim().x + for i in index:stride:p + view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b + end + return nothing +end + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all batch of another 4D matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> + julia> a = rand(2,2,3) # 3-batches + julia> b = rand(2,2,4,3) # 3-batches + julia> r = GeneralUtils.matMul_3Dto4D_batchwise(a, b); + julia> size(r) + (2, 2, 4, 3) +""" +function matMul_3Dto4D_batchwise(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if asize[end] != bsize[end] + error("batch number of a and b must be equal") + end + if resultStorage === nothing + resultStorage = zeros(bsize[1], bsize[2], bsize[3], asize[3]) + end + for i in 1:asize[3] + view(resultStorage, :, :, :, i) .= a[:, :, i] .* b[:, :, :, i] + end + return resultStorage +end + +#TODO +function matMul_4Dto4D_batchwise(a::Array, b::Array; + resultStorage::Union{Array, Nothing}=nothing) + error("this function is under constrution") + asize = [size(a)...] + bsize = [size(b)...] + if asize[end] != bsize[end] + error("batch number of a and b must be equal") + end + if resultStorage === nothing + resultStorage = zeros(bsize[1], bsize[2], bsize[3], asize[3]) + end +end + +""" assign matrix b to matrix a according to matrix b's CartesianIndex +""" +cartesianAssign!(a::CuArray, b::CuArray) = @cuda cartesianAssign!(a, b) + +function cartesianAssign!(a, b) + for (i, v) in enumerate(b) + a[CartesianIndices(b)[i].I...] = v + end + return nothing +end + +function sumAlongDim3(a::Array) + totalDim = length(size(a)) + + if totalDim == 3 + d1, d2, d3 = size(a) + r = zeros(1, 1, d3) + for i in 1:d3 + view(r, 1, 1, i) .= sum(a[:, :, i]) + end + elseif totalDim == 4 + d1, d2, d3, d4 = size(a) + r = zeros(1, 1, d3, d4) + for j in 1:d4 + for i in 1:d3 + view(r, 1, 1, i, j) .= sum(a[:, :, i, j]) + end + end + else + error("this condition is not define yet") + end + + return r +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.1/test/etc.jl b/previousVersion/0.0.1/test/etc.jl new file mode 100644 index 0000000..e69de29 diff --git a/previousVersion/0.0.2/Manifest.toml b/previousVersion/0.0.2/Manifest.toml new file mode 100644 index 0000000..ebad44a --- /dev/null +++ b/previousVersion/0.0.2/Manifest.toml @@ -0,0 +1,911 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.1" +manifest_format = "2.0" +project_hash = "91349a0eb57aed703e8465e44731cca768229f09" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "8bc0aaec0ca548eb6cf5f0d7d16351650c1ee956" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.3.2" +weakdeps = ["ChainRulesCore"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.6.2" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgCheck]] +git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.3.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "dbf84058d0a8cbbadee18d25cf606934b22d7c66" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.4.2" + +[[deps.BangBang]] +deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] +git-tree-sha1 = "e28912ce94077686443433c2800104b061a827ed" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.3.39" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.CEnum]] +git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.2" + +[[deps.CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "442d989978ed3ff4e174c928ee879dc09d1ef693" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "4.3.2" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] +git-tree-sha1 = "498f45593f6ddc0adff64a9310bb6710e851781b" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "0.5.0+1" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "0.2.2" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "5248d9c45712e51e27ba9b30eebec65658c6ce29" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.6.0+0" + +[[deps.CUDNN_jll]] +deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "2918fbffb50e3b7a0b9127617587afa76d4276e8" +uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" +version = "8.8.1+0" + +[[deps.Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] +git-tree-sha1 = "1cdf290d4feec68824bfb84f4bfc9f3aba185647" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.51.1" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.16.0" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[deps.Compat]] +deps = ["UUIDs"] +git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.6.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.2+0" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + + [deps.CompositionsBase.weakdeps] + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.ConstructionBase]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "738fec4d684a9a6ee9598a8bfee305b26831f28c" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.2" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.ContextVariablesX]] +deps = ["Compat", "Logging", "UUIDs"] +git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" +uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" +version = "0.1.3" + +[[deps.DataAPI]] +git-tree-sha1 = "e8119c1a33d267e16108be441a287a6981ba1630" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.14.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "d1fff3a548102f48987a52a2e0d114fa97d730f0" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.13" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] +git-tree-sha1 = "eead66061583b6807652281c0fbf291d7a9dc497" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.90" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.DualNumbers]] +deps = ["Calculus", "NaNMath", "SpecialFunctions"] +git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" +uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" +version = "0.6.8" + +[[deps.ExprTools]] +git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.9" + +[[deps.FLoops]] +deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] +git-tree-sha1 = "ffb97765602e3cbe59a0589d237bf07f245a8576" +uuid = "cc61a311-1640-44b5-9fba-1b764f453329" +version = "0.2.1" + +[[deps.FLoopsBase]] +deps = ["ContextVariablesX"] +git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" +uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" +version = "0.1.1" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "fc86b4fd3eff76c3ce4f5e96e2fdfa6282722885" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.0.0" + +[[deps.Flux]] +deps = ["Adapt", "CUDA", "ChainRulesCore", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "NNlibCUDA", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote", "cuDNN"] +git-tree-sha1 = "3e2c3704c2173ab4b1935362384ca878b53d4c34" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.13.17" + + [deps.Flux.extensions] + AMDGPUExt = "AMDGPU" + FluxMetalExt = "Metal" + + [deps.Flux.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "00e252f4d706b3d55a8863432e742bf5717b498d" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.35" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Functors]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "478f8c3145bb91d82c2cf20433e8c1b30df454cc" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.4.4" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] +git-tree-sha1 = "2e57b4a4f9cc15e85a24d603256fe08e527f48d1" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "8.8.1" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.5" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "cb090aea21c6ca78d59672a7e7d13bd56d09de64" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.20.3" + +[[deps.HypergeometricFunctions]] +deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "84204eae2dd237500835990bcade263e27674a93" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.16" + +[[deps.IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "eac00994ce3229a464c2847e956d77a2c64ad3a5" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.10" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.4.1" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "SnoopPrecompile", "StructTypes", "UUIDs"] +git-tree-sha1 = "84b10656a41ef564c39d2d477d7236966d2b5683" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.12.0" + +[[deps.JuliaVariables]] +deps = ["MLStyle", "NameResolution"] +git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" +uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" +version = "0.2.4" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "b48617c5d764908b5fac493cd907cf33cc11eec1" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.6" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "5007c1421563108110bbd57f63d8ad4565808818" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "5.2.0" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "1222116d7313cdefecf3d45a2bc1a89c4e7c9217" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.22+0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "0a1b7c2863e44523180fdb3146534e265a91870b" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.23" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.MLStyle]] +git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" +uuid = "d8e11817-5142-5d16-987a-aa16d5891078" +version = "0.4.17" + +[[deps.MLUtils]] +deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] +git-tree-sha1 = "3504cdb8c2bc05bde4d4b09a81b01df88fcbbba0" +uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" +version = "0.4.3" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.10" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.MicroCollections]] +deps = ["BangBang", "InitialValues", "Setfield"] +git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.1.4" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.1.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] +git-tree-sha1 = "72240e3f5ca031937bd536182cb2c031da5f46dd" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.8.21" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + +[[deps.NNlibCUDA]] +deps = ["Adapt", "CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics", "cuDNN"] +git-tree-sha1 = "f94a9684394ff0d325cc12b06da7032d8be01aaf" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.2.7" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.0.2" + +[[deps.NameResolution]] +deps = ["PrettyPrint"] +git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" +uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" +version = "0.1.5" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OneHotArrays]] +deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] +git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c" +uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" +version = "0.2.4" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "6a01f65dd8583dee82eecc2a19b0ff21521aa749" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.2.18" + +[[deps.OrderedCollections]] +git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.0" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "67eae2738d63117a196f497d7db789821bce61d1" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.17" + +[[deps.Parsers]] +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.5.8" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.2" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.0" + +[[deps.PrettyPrint]] +git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" +uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" +version = "0.2.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.8.2" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.6.1" + +[[deps.RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.5.3" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.0+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.1" + +[[deps.ShowCases]] +git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" +uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" +version = "0.1.0" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.SnoopPrecompile]] +deps = ["Preferences"] +git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.3" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "a4ada03f999bd01b3a25dcaa30b2d929fe537e00" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.1.0" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "ef28127915f4229c971eb43f3fc075dd3fe91880" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.2.0" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] +git-tree-sha1 = "832afbae2a45b4ae7e831f86965469a24d1d8a83" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.5.26" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.6.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.0" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.0" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StructArrays]] +deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] +git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.15" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.10.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.10.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.23" + +[[deps.Transducers]] +deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] +git-tree-sha1 = "a66fb81baec325cf6ccafa243af573b031e87b00" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.77" + + [deps.Transducers.extensions] + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "ea37e6066bf194ab78f4e747f5245261f17a7175" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.1.2" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "5be3ddb88fc992a7d8ea96c3f10a49a7e98ebc7b" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.62" + + [deps.Zygote.extensions] + ZygoteColorsExt = "Colors" + ZygoteDistancesExt = "Distances" + ZygoteTrackerExt = "Tracker" + + [deps.Zygote.weakdeps] + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "977aed5d006b840e2e40c0b48984f7463109046d" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.3" + +[[deps.cuDNN]] +deps = ["CEnum", "CUDA", "CUDNN_jll"] +git-tree-sha1 = "f65490d187861d6222cb38bcbbff3fd949a7ec3e" +uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" +version = "1.0.4" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/previousVersion/0.0.2/Project.toml b/previousVersion/0.0.2/Project.toml new file mode 100644 index 0000000..d09189f --- /dev/null +++ b/previousVersion/0.0.2/Project.toml @@ -0,0 +1,11 @@ +name = "GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +authors = ["tonaerospace "] +version = "0.1.0" + +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" diff --git a/previousVersion/0.0.2/src/GeneralUtils.jl b/previousVersion/0.0.2/src/GeneralUtils.jl new file mode 100644 index 0000000..5cd0f5f --- /dev/null +++ b/previousVersion/0.0.2/src/GeneralUtils.jl @@ -0,0 +1,81 @@ +module GeneralUtils + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue + + +include("interface.jl") +using .interface + + +#------------------------------------------------------------------------------------------------100 + +""" version 0.0.2 + Todo: + - [] + + Change from version: + - + + All features + +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # generalUtils \ No newline at end of file diff --git a/previousVersion/0.0.2/src/interface.jl b/previousVersion/0.0.2/src/interface.jl new file mode 100644 index 0000000..74e7edd --- /dev/null +++ b/previousVersion/0.0.2/src/interface.jl @@ -0,0 +1,841 @@ +module interface + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, replaceMoreThan, + replaceLessThan, cartesianAssign!, sumAlongDim3, matMul_3Dto3D_manyTo1batch, + matMul_3Dto4D_batchwise + +using JSON3, DataStructures, Distributions, Flux, CUDA + +#------------------------------------------------------------------------------------------------100 + +noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) +findNotZero(x::AbstractVector) = findall( (!iszero).(x) ) +replaceMoreThan(i, target) = i > target ? target : i +replaceLessThan(i, target) = i < target ? target : i +precision(x::Array{<:Array}) = ( std(mean.(x)) / mean(mean.(x)) ) * 100 +precision(x::Array) = std(x) / mean(x) * 100 +replaceAt!(x::AbstractVector, ind::Number, value::Number) = x[ind] = value +notZero(x::AbstractVector) = (!iszero).(x) +Zero(x::AbstractVector) = iszero.(x) +isNan(x::AbstractVector) = isnan.(x) +isInf(x::Number) = abs(x) === Inf +isInf(x::AbstractVector) = isinf.(x) +isLess(x::AbstractVector, target::Number) = isless.(x, target) +isMore(x::Number, target::Number) = x > target +isMore(x::AbstractVector, target::Number) = isMore.(x, target) +absolute(x::AbstractVector) = abs.(x) +vecEleMul(x::AbstractVector, y::AbstractVector) = x .* y +vecEleMul(x::Number, y::AbstractVector) = x .* y +expDecay(initialValue::Number, decayFactor::Number, timePass::Number) = + initialValue * (1 - decayFactor)^timePass +mul!(x::AbstractVector, y::AbstractVector) = x .*= y +mul(x::AbstractVector, y::AbstractVector) = x .* y + +ReLu(x::Number) = max(0, x) + +updateVector!(x::AbstractVector, target::Number) = x .= target +updateVector!(x::AbstractVector, target::AbstractArray) = x .= target + +function selectAdd!(x::AbstractVector, ind::AbstractVector, value::AbstractVector) + @. x = x + (ind * value) +end + + +""" findIndex(input::String, target::Char) + + Find target index inside a collection. + Return 1) Bool array of matched target + 2) CartesianIndex of every matched target +""" +function findIndex(input::String, target::Char) + match_position = [] + for i in input + if i == target + append!(match_position, 1) + else + append!(match_position, 0) + end + end + match_index = findall(isequal.(match_position, 1)) + + return match_position, match_index +end + +function findIndex(input::Array, target::Number) + match_position = isequal.(input, target) + match_index = findall(match_position) + return match_position, match_index +end + +# function findIndex(input::Array, target::Array) +# match_position = isone.(zeros(length(input))) +# for i in target +# match_position = match_position + isequal.(input, i) +# end + +# match_position = replaceMoreThan.(match_position, 1) +# match_index = findall(isone.(match_position)) # Findall donot work with Int64 vector [1, 0, 0, 1]. +# # It only works with BitVector. isone() converts Int64 vector [1, 0, 0, 1] into +# # BitVector [1, 0, 0, 1] + +# return match_position, match_index +# end + +function findIndex(input::Array, target::Symbol) + match_position = isequal.(input, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::String) + match_position = isequal.(collection, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::Array{String}) + match_position = nothing + match_index = nothing + + for i in target + match_pos = isequal.(collection, i) + match_ind = findall(match_pos) + + if match_position === nothing + match_position = match_pos + + else + match_position = hcat(match_position, match_pos) + end + + if match_index === nothing + match_index = match_ind + else + match_index = hcat(match_index, match_ind) + end + end + + return match_position, match_index +end + +function findIndex(collection::OrderedDict, target::Symbol) + collection_keys = keys(collection) + collection_keys_array = [i for i in collection_keys] + match_position = isequal.(collection_keys_array, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findMax(collection::AbstractVector) + maxValue, maxIndex = findmax(collection) + matchPosition = isequal.(collection, maxValue) + return maxValue, maxIndex, matchPosition +end + + + +""" read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + + with multiple text file in a folder, + this function read x_th text file in a folder (filename is sorted by OS) + + # Example + utils.read_textfile_by_index(cleaned_data_path, 2) + read 2nd txt file in a folder +""" +function read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + if isdir(folder_path) + filenumber = length(readdir(folder_path)) + + if read_file_number > filenumber + error("you specified read_file_number = $read_file_number which is out + of range, the cleaned data folder has only $filenumber files") + return nothing, nothing, nothing + else + content = 0 + # open each file in the directory and read + filename = readdir(folder_path, join=true, sort=false)[read_file_number] + f = open(filename) + content = readlines(f) + # content = read(f) + close(f) + end + + return read_file_number, filename, content + else + error("ERROR no file or folder at $folder_path") + return nothing, nothing, nothing + end + +end + +#------------------------------------------------------------------------------------------------100 + +""" Array_to_JSON3_str(data::AbstractArray) + + encode Array to JSON3 String + + # Example + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] + json3_str = Array_to_JSON3_str(a) + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} +""" +function Array_to_JSON3_str(data::AbstractArray) + d = Dict("Array"=> data, "size"=>size(data)) + json3_str = JSON3.write(d) + return json3_str +end + +#------------------------------------------------------------------------------------------------100 + +""" JSON3_str_to_Array(json3_str::String) + + decode JSON3 String to Array + + # Example + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} + a = JSON3_str_to_Array(json3_str) + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] +""" +function JSON3_str_to_Array(json3_str::String) + d = JSON3.read(json3_str) + array = reshape(Array(d.Array), (d.size[1], d.size[2])) + return array +end + +#------------------------------------------------------------------------------------------------100 + +""" Convert JSON3.read object to OrderedDict + + # Example + dict = dictionary(["a"=>4, "b"=>6]) + OrDict = OrderedDict(dict) + jsonString = JSON3.write(OrDict) # use jsonString to exchange. One can save it to file or send it thru pub/sub + jsonObject = JSON3.read(jsonString) + OrDict2 = JSON3read_to_OrDict(jsonObject) # example here + Adict2 = dictionary(OrDict2) + + Andyferris's github https://github.com/andyferris/Dictionaries.jl +""" +function JSON3read_to_OrDict(x) + dict = OrderedDict() + for (k, v) in x + k = string(k) + dict[k] = v + end + return dict +end + +#------------------------------------------------------------------------------------------------100 + +""" + print time of cpu executtion at the line inwhich this macro is used +""" +macro timeline(expr) + quote + print("line ", $(__source__.line), ": ") + @time $(esc(expr)) + end +end + + +batchindex(batch_counter::Number, batch_size::Number; offset=0) = + (offset + (batch_counter-1) * batch_size + 1) : offset + (batch_counter * batch_size) + +function flip_true_false(x::Bool) + if x == true + x = false + elseif x == false + x = true + else + error("undefined condition line $(@__LINE__)") + end + + return x +end + +function flip_true_false(x::Int) + if x == 1 + x = 0 + elseif x == 0 + x = 1 + else + throw("not define input of type $(typeof(x)) yet") + end + + return x +end + +""" + Return drawed index + # Example + drawed_index = randomWithProb([0.5, 0.2, 0.3]) + +""" +randomWithProb(probability::AbstractVector) = rand(Distributions.Categorical(probability)) # return drawed index + +""" + Draw from choices according to its probability. + Probability range is 0.0 to 1.0 and all probability must summed up to 1 + (may get probability from NNlib's softmax function) + + # Example + + draw = draw_choices([true, false, nothing], [0.5, 0.2, 0.3]) +""" +function randomChoiceWithProb(choices::Array, probability::Array) + if length(choices) != length(probability) + error("random is not possible, choices array length != probability array length") + elseif sum(probability) != 1.0 + error("probability does not sum to 1.0") + end + + return choices[randomWithProb(probability)] +end + +function randomChoiceOnTarget(target::Number, targetMatch::Number, choices::AbstractVector, + probability::AbstractVector) + if length(choices) != length(probability) + throw("random is not possible, choices array length != probability array length") + end + return target == targetMatch ? randomChoiceWithProb(choices, probability) : target + # dist = Distributions.Categorical(probability) + # draw_result = choices[rand(dist)] +end + +function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector, + probability::AbstractVector) + return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,)) +end + +function linearly_weighted_avg(a::Array) + total = 0.0 + for (i, v) in enumerate(a) + total = total + (i * v) + end + + return total / sum(a) +end + + +""" Convert String that is holded inside a variable to Symbol +# Example +x = "hello" # x is a variable holding String "hello" \n +y = variable_to_symbol(x) # y holds :hello +""" +function variable_str_to_symbol(variable) + semi = :($variable) + symbol = Symbol(semi) + + return symbol +end + + +""" get useable type of specified fieldname inside a composite struct + + # Example + julia> @Base.kwdef mutable struct some_struct + a::Union{Bool, Nothing} = nothing + b::Union{Float64, Nothing} = nothing + c::Union{Int64, AbstractFloat} = 3.5 + d::Union{String, Nothing} = nothing + end + + julia> a = some_struct() + julia> fieldname_useable_type(some_struct, :c) =result=> [Int64, Float64] +""" +function fieldname_useable_type(somestruct, fieldname::Symbol; + test_types=[2.0, 2, true, :h, "str", 'c', missing, nothing])::Vector{DataType} + new_instance = somestruct() + useable_type = [] + + for i in test_types + try + new_instance.:($fieldname) = i + type = typeof(new_instance.:($fieldname)) + if type ∉ useable_type + push!(useable_type, type) + end + catch + + end + end + + return useable_type +end + + +function randomNoRepeat(drawOptions::Array, draw_number::Integer; + exclude_list::Union{AbstractArray,Nothing}=nothing) + draw_option = copy(drawOptions) + draw_option = isnothing(exclude_list) ? draw_option : + filter!(x -> x ∉ exclude_list, draw_option) + shuffle!(draw_option) + drawed_items = [] + while length(drawed_items) < draw_number + push!(drawed_items, pop!(draw_option)) + end + return drawed_items +end + +""" using cron to schedule backup job by + 1. sudo nano /etc/crontab <<< this is a system-wide cron file + 2. to execute julia file @ 2.00am everyday add the following line at the buttom of the file + 0 2 * * * root julia-1.7 /home/syncthing_backup_script.jl + + Requirements using Dates +""" +function folderBackup(sourceFolderAbsolutePath::String, # absolute path to folder to be backuped + backupFolderAbsolutePath::String; # absolute path to folder used to store backup file + totalBackupFiles::Integer=7, # total backup file, the oldest will be deleted + containerName::Union{Array{String}, Nothing}=nothing) # container using source_folder + + sep = (Sys.iswindows() ? "\\" : '/') + + if sourceFolderAbsolutePath[end] == sep + sourceFolderAbsolutePath = sourceFolderAbsolutePath[1:end-1] + end + if backupFolderAbsolutePath[end] != sl + backupFolderAbsolutePath = backupFolderAbsolutePath * sep + end + + if isdir(backupFolderAbsolutePath) + else + mkpath(backupFolderAbsolutePath) + end + + # stop running docker container service + if containerName !== nothing + println("stop running services") + for i in containerName + try run(`docker stop $i`) catch; end + sleep(10) # wait for services to stop + end + end + + # do backup + println("doing backup now") + timestamp = string(Dates.now()) + name = split(sourceFolderAbsolutePath, sep)[end] * "--" + filename = name * timestamp * ".zip" # resulting compressed filename + run(`chmod -R a+rwx $sourceFolderAbsolutePath`) + # zip -r [destination+filename] [source folder to be zipped] + run(`zip -r $(backupFolderAbsolutePath * filename) $sourceFolderAbsolutePath`) + + # check if total backup file is more than user specified, if yes, delete the oldest backup + backupFiles = readdir(backupFolderAbsolutePath) + + while length(backupFiles) > totalBackupFiles + run(`rm $(backupFolderAbsolutePath * backupFiles[1])`) + backupFiles = readdir(backupFolderAbsolutePath) + end + + # start docker services + if containerName !== nothing + println("start services") + for i in containerName + try run(`docker start $i`) catch; end + sleep(10) # wait for services to stop + end + end +end + +function lowerclip!(data::AbstractVector, lowerbound::Number) + replace!(x -> x < lowerbound ? lowerbound : x, data) +end + +function upperclip!(data::AbstractVector, upperbound::Number) + replace!(x -> x > upperbound ? upperbound : x, data) +end + +function normalise(x::AbstractArray, mu, std) + ϵ = oftype(x[1], 1e-5) + μ = mu + # σ = std(x, dims=dims, mean=μ, corrected=false) # use this when Zygote#478 gets merged + σ = std + return (x .- μ) ./ (σ .+ ϵ) +end + +function minMaxScaler(x::AbstractVector) + min = findmin(x)[1] + max = findmax(x)[1] + + scaler(a::Number, min::Number, max::Number) = (a-min) / (max-min) + return scaler.(x, min, max) +end + +""" a = [-1e200, -1e-200, 1e200, 1e-200] \n + result = vtclamp.(a, 1e-6, 1e6, -1e6, -1e-6) +""" +function customclamp(x::Number, poslo::Number, poshi::Number, + neglo::Number, neghi::Number) + signx = sign(x) + if signx == -1 + if neghi < x < 0 + return neghi + elseif x < neglo + return neglo + else + return x + end + elseif signx == +1 + if poshi < x + return poshi + elseif 0 < x < poslo + return poslo + else + return x + end + end +end + +function unitVec(x::AbstractVector) + y = √(sum(x.^2)) + return x./y +end + +function replaceAt!(x::AbstractVector, ind::AbstractVector, value::Number) + for i in ind + x[i] = value + end +end + +function signbitVec(x::AbstractVector) + sign = signbit.(x) * 1 + signVec = replace(s -> s == 0 ? -1 : s, sign) + return signVec +end + +function deleteall!(x::AbstractVector) + for i in 1:length(x) + deleteat!(x, 1) + end +end + +""" Select specific range of vectors in a dict, return a new dict +# Example +dict = Dict(:a => [1:5...], + :b => [6:10...]) + +call -> selectRange(dict, 1:3) +return -> Dict{Any, Any} with 2 entries: + :a => [1, 2, 3] + :b => [6, 7, 8] +""" +function selectRange(d::Dict{Symbol, <:AbstractVector}, range) + newDict = Dict{Symbol, AbstractVector}() + for (k, v) in d + newDict[k] = v[range] + end + + return newDict +end + +""" Assign value to a given Dict by array of keys + + # Example + d = Dict( + :a1=> Dict(:c=> 5), + :a2=> Dict( + :k=> 10, + :b=> Dict( + :s=> "target", + ) + ) + ) + index = [:a2, :b, :s] \n + assignDict!(d, [:a2, :b, :s], "wow") + + return 1 if no target key in a given dict. +""" +function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign) + wd = nothing + for i in accessArray + println(i) + if i != accessArray[end] + if wd === nothing && haskey(dict, i) + wd = Ref(dict[i]) + elseif wd.x !== nothing && haskey(wd.x, i) + wd = Ref(wd.x[i]) + else + return 1 # error, no target key in a given dict. + end + else + wd.x[i] = valueToAssign + return 0 + end + end +end + +""" convert hour(0-23), minute(0-59) into julia time object +# Example + time +""" +function iTime(h::Integer, m::Integer) + if h == 0 + h = 12 + ampm = "am" + elseif 1 <= h <= 11 + ampm = "am" + elseif h == 12 + ampm = "pm" + elseif 13 <= h <= 23 + h = h - 12 + ampm = "pm" + else + error("hour out of range") + end + + m = m < 10 ? "0$m" : m + t = "$h:$m$ampm" + + return Time(t, "HH:MMp") +end + +""" replace a number according to the limit + if value is lower than lowerbound return lowerbound replacement value + if value is more than upperbound return upperbound replacement value + + # Example + limitvalue(4, (-5 => 0), (5 => 5)) +""" +function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair) + lwLimit, lwReplace = lowerbound + upLimit, upReplace = upperbound + + if v < lwLimit + v = lwReplace + elseif v > upLimit + v = upReplace + else + end + return v +end + +""" assign matrix b to matrix a according to matrix b's CartesianIndex +""" +cartesianAssign!(a::CuArray, b::CuArray) = @cuda cartesianAssign!(a, b) + +function cartesianAssign!(a, b) + for (i, v) in enumerate(b) + a[CartesianIndices(b)[i].I...] = v + end + return nothing +end + +function sumAlongDim3(a::Array) + totalDim = length(size(a)) + + if totalDim == 3 + d1, d2, d3 = size(a) + r = zeros(1, 1, d3) + for i in 1:d3 + view(r, 1, 1, i) .= sum(a[:, :, i]) + end + elseif totalDim == 4 + d1, d2, d3, d4 = size(a) + r = zeros(1, 1, d3, d4) + for j in 1:d4 + for i in 1:d3 + view(r, 1, 1, i, j) .= sum(a[:, :, i, j]) + end + end + else + error("this condition is not define yet") + end + + return r +end + + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all slice of 3D another matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> input = rand(32, 32, 128) # batch at 3rd dim + julia> weight = rand(32, 32, 1024) + julia> r = matMul_3Dto3D_manyTo1batch(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul_3Dto3D_manyTo1batch(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if resultStorage === nothing + resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) + end + + c = [slice .* b for slice in eachslice(a, dims=3)] + resultStorage .= cat(c..., dims=4) + + return resultStorage +end + +# function matMul_3Dto3D_manyTo1batch(a::CuArray, b::CuArray; # XXX working code +# resultStorage::Union{CuArray, Nothing}=nothing, threads=256) +# asize = [size(a)...] +# bsize = [size(b)...] +# if resultStorage === nothing +# resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) |> gpu +# end +# CUDA.@sync begin +# @cuda threads=threads matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) +# end +# return resultStorage +# end + +# function matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) # XXX working code +# _, _, _, p = size(resultStorage) + +# index = threadIdx().x # this example only requires linear indexing, so just use `x` +# stride = blockDim().x +# for i in index:stride:p +# view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b +# end +# return nothing +# end + +""" GPU kernel +""" +function matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) + _, _, batch = size(a) # This kernel use 1 thread per batch + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + if i <= batch # guard against unused threads to accessing memory out of bound + # @cuprintln("thread $i") + view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b + end + return nothing +end + +""" GPU version of batchMatEleMul + + Example + julia> using Flux, CUDA + julia> device = Flux.CUDA.functional() ? gpu : cpu + julia> if device == gpu CUDA.device!(0) end + julia> input = rand(32, 32, 128) |> gpu; # 128-batches + julia> weight = rand(32, 32, 1024) |> gpu; # 1-batch + julia> r = matMul_3Dto3D_manyTo1batch(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul_3Dto3D_manyTo1batch(a::CuArray, b::CuArray; + resultStorage::Union{CuArray, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if resultStorage === nothing + resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) |> gpu + end + + kernel = @cuda launch=false matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) + config = launch_configuration(kernel.fun) + + # threads to be launched. Since one can't launch exact thread number the kernel needs, + # one just launch threads more than this kernel needs then use a guard inside the kernel + # to prevent unused threads to access memory. + threads = min(1024, config.threads) # most NVIDIA gpu has 1024 threads per block + blocks = cld(asize[3], threads) # This kernel use 1 thread per batch + + CUDA.@sync begin + kernel(a, b, resultStorage; threads, blocks) + end + return resultStorage +end + + + + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all batch of another 4D matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> + julia> a = rand(2,2,3) # 3-batches + julia> b = rand(2,2,4,3) # 3-batches + julia> r = GeneralUtils.matMul_3Dto4D_batchwise(a, b); + julia> size(r) + (2, 2, 4, 3) +""" +function matMul_3Dto4D_batchwise(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if asize[end] != bsize[end] + error("batch number of a and b must be equal") + end + if resultStorage === nothing + resultStorage = zeros(bsize[1], bsize[2], bsize[3], asize[3]) + end + for i in 1:asize[3] + view(resultStorage, :, :, :, i) .= a[:, :, i] .* b[:, :, :, i] + end + return resultStorage +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.2/test/etc.jl b/previousVersion/0.0.2/test/etc.jl new file mode 100644 index 0000000..e69de29 diff --git a/previousVersion/0.0.3/Manifest.toml b/previousVersion/0.0.3/Manifest.toml new file mode 100644 index 0000000..e23be4a --- /dev/null +++ b/previousVersion/0.0.3/Manifest.toml @@ -0,0 +1,911 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.2" +manifest_format = "2.0" +project_hash = "168be3100b37ab8a45dff337b2c1caa16b5ecd26" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "8bc0aaec0ca548eb6cf5f0d7d16351650c1ee956" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.3.2" +weakdeps = ["ChainRulesCore"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.6.2" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgCheck]] +git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.3.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "dbf84058d0a8cbbadee18d25cf606934b22d7c66" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.4.2" + +[[deps.BangBang]] +deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] +git-tree-sha1 = "e28912ce94077686443433c2800104b061a827ed" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.3.39" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.CEnum]] +git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.2" + +[[deps.CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "442d989978ed3ff4e174c928ee879dc09d1ef693" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "4.3.2" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] +git-tree-sha1 = "498f45593f6ddc0adff64a9310bb6710e851781b" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "0.5.0+1" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "0.2.2" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "5248d9c45712e51e27ba9b30eebec65658c6ce29" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.6.0+0" + +[[deps.CUDNN_jll]] +deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "2918fbffb50e3b7a0b9127617587afa76d4276e8" +uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" +version = "8.8.1+0" + +[[deps.Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] +git-tree-sha1 = "1cdf290d4feec68824bfb84f4bfc9f3aba185647" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.51.1" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.16.0" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[deps.Compat]] +deps = ["UUIDs"] +git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.6.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.5+0" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + + [deps.CompositionsBase.weakdeps] + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.ConstructionBase]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "738fec4d684a9a6ee9598a8bfee305b26831f28c" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.2" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.ContextVariablesX]] +deps = ["Compat", "Logging", "UUIDs"] +git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" +uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" +version = "0.1.3" + +[[deps.DataAPI]] +git-tree-sha1 = "e8119c1a33d267e16108be441a287a6981ba1630" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.14.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "d1fff3a548102f48987a52a2e0d114fa97d730f0" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.13" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] +git-tree-sha1 = "eead66061583b6807652281c0fbf291d7a9dc497" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.90" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.DualNumbers]] +deps = ["Calculus", "NaNMath", "SpecialFunctions"] +git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" +uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" +version = "0.6.8" + +[[deps.ExprTools]] +git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.9" + +[[deps.FLoops]] +deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] +git-tree-sha1 = "ffb97765602e3cbe59a0589d237bf07f245a8576" +uuid = "cc61a311-1640-44b5-9fba-1b764f453329" +version = "0.2.1" + +[[deps.FLoopsBase]] +deps = ["ContextVariablesX"] +git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" +uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" +version = "0.1.1" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "fc86b4fd3eff76c3ce4f5e96e2fdfa6282722885" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.0.0" + +[[deps.Flux]] +deps = ["Adapt", "CUDA", "ChainRulesCore", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "NNlibCUDA", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote", "cuDNN"] +git-tree-sha1 = "3e2c3704c2173ab4b1935362384ca878b53d4c34" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.13.17" + + [deps.Flux.extensions] + AMDGPUExt = "AMDGPU" + FluxMetalExt = "Metal" + + [deps.Flux.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "00e252f4d706b3d55a8863432e742bf5717b498d" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.35" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Functors]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "478f8c3145bb91d82c2cf20433e8c1b30df454cc" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.4.4" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] +git-tree-sha1 = "2e57b4a4f9cc15e85a24d603256fe08e527f48d1" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "8.8.1" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.5" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "cb090aea21c6ca78d59672a7e7d13bd56d09de64" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.20.3" + +[[deps.HypergeometricFunctions]] +deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "84204eae2dd237500835990bcade263e27674a93" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.16" + +[[deps.IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "eac00994ce3229a464c2847e956d77a2c64ad3a5" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.10" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.4.1" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "SnoopPrecompile", "StructTypes", "UUIDs"] +git-tree-sha1 = "84b10656a41ef564c39d2d477d7236966d2b5683" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.12.0" + +[[deps.JuliaVariables]] +deps = ["MLStyle", "NameResolution"] +git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" +uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" +version = "0.2.4" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "b48617c5d764908b5fac493cd907cf33cc11eec1" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.6" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "5007c1421563108110bbd57f63d8ad4565808818" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "5.2.0" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "1222116d7313cdefecf3d45a2bc1a89c4e7c9217" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.22+0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "0a1b7c2863e44523180fdb3146534e265a91870b" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.23" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.MLStyle]] +git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" +uuid = "d8e11817-5142-5d16-987a-aa16d5891078" +version = "0.4.17" + +[[deps.MLUtils]] +deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] +git-tree-sha1 = "3504cdb8c2bc05bde4d4b09a81b01df88fcbbba0" +uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" +version = "0.4.3" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.10" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.MicroCollections]] +deps = ["BangBang", "InitialValues", "Setfield"] +git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.1.4" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.1.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] +git-tree-sha1 = "72240e3f5ca031937bd536182cb2c031da5f46dd" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.8.21" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + +[[deps.NNlibCUDA]] +deps = ["Adapt", "CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics", "cuDNN"] +git-tree-sha1 = "f94a9684394ff0d325cc12b06da7032d8be01aaf" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.2.7" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.0.2" + +[[deps.NameResolution]] +deps = ["PrettyPrint"] +git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" +uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" +version = "0.1.5" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OneHotArrays]] +deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] +git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c" +uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" +version = "0.2.4" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "6a01f65dd8583dee82eecc2a19b0ff21521aa749" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.2.18" + +[[deps.OrderedCollections]] +git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.0" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "67eae2738d63117a196f497d7db789821bce61d1" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.17" + +[[deps.Parsers]] +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.5.8" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.2" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.2" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.0" + +[[deps.PrettyPrint]] +git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" +uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" +version = "0.2.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.8.2" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.6.1" + +[[deps.RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.5.3" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.0+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.1" + +[[deps.ShowCases]] +git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" +uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" +version = "0.1.0" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.SnoopPrecompile]] +deps = ["Preferences"] +git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.3" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "a4ada03f999bd01b3a25dcaa30b2d929fe537e00" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.1.0" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "ef28127915f4229c971eb43f3fc075dd3fe91880" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.2.0" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] +git-tree-sha1 = "832afbae2a45b4ae7e831f86965469a24d1d8a83" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.5.26" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.6.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.0" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.0" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StructArrays]] +deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] +git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.15" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.10.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.10.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.23" + +[[deps.Transducers]] +deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] +git-tree-sha1 = "a66fb81baec325cf6ccafa243af573b031e87b00" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.77" + + [deps.Transducers.extensions] + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "ea37e6066bf194ab78f4e747f5245261f17a7175" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.1.2" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "5be3ddb88fc992a7d8ea96c3f10a49a7e98ebc7b" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.62" + + [deps.Zygote.extensions] + ZygoteColorsExt = "Colors" + ZygoteDistancesExt = "Distances" + ZygoteTrackerExt = "Tracker" + + [deps.Zygote.weakdeps] + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "977aed5d006b840e2e40c0b48984f7463109046d" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.3" + +[[deps.cuDNN]] +deps = ["CEnum", "CUDA", "CUDNN_jll"] +git-tree-sha1 = "f65490d187861d6222cb38bcbbff3fd949a7ec3e" +uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" +version = "1.0.4" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/previousVersion/0.0.3/Project.toml b/previousVersion/0.0.3/Project.toml new file mode 100644 index 0000000..0fc70d3 --- /dev/null +++ b/previousVersion/0.0.3/Project.toml @@ -0,0 +1,12 @@ +name = "GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +authors = ["tonaerospace "] +version = "0.1.0" + +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/previousVersion/0.0.3/src/GeneralUtils.jl b/previousVersion/0.0.3/src/GeneralUtils.jl new file mode 100644 index 0000000..23c07c9 --- /dev/null +++ b/previousVersion/0.0.3/src/GeneralUtils.jl @@ -0,0 +1,81 @@ +module GeneralUtils + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue + + +include("interface.jl") +using .interface + + +#------------------------------------------------------------------------------------------------100 + +""" version 0.0.3 + Todo: + - [*1] cartesianAssign for different matrix dimension + + Change from version: 0.0.2 + - + + All features + +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # generalUtils \ No newline at end of file diff --git a/previousVersion/0.0.3/src/interface.jl b/previousVersion/0.0.3/src/interface.jl new file mode 100644 index 0000000..ca3f56b --- /dev/null +++ b/previousVersion/0.0.3/src/interface.jl @@ -0,0 +1,1108 @@ +module interface + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, replaceMoreThan, + replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul_3Dto3D_manyTo1batch, + matMul_3Dto4D_batchwise, isNotEqual, linear_to_cartesian, vectorMax, + multiply_last, multiply_random_elements, replaceElements, replaceElements!, isBetween, + isLess, allTrue, getStringBetweenCharacters + +using JSON3, DataStructures, Distributions, Flux, CUDA, Random + +#------------------------------------------------------------------------------------------------100 + +noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) +findNotZero(x::AbstractVector) = findall( (!iszero).(x) ) +replaceMoreThan(x, target, replaceValue) = x > target ? replaceValue : x +replaceMoreThan(x, target, a, b) = x > target ? a : b +replaceLessThan(x, target, replaceValue) = x < target ? replaceValue : x +replaceLessThan(x, target, a, b) = x < target ? a : b +replaceBetween(x, lowerbound, upperbound, replaceValue) = lowerbound < x < upperbound ? replaceValue : x +precision(x::Array{<:Array}) = ( std(mean.(x)) / mean(mean.(x)) ) * 100 +precision(x::Array) = std(x) / mean(x) * 100 +replaceAt!(x::AbstractVector, ind::Number, value::Number) = x[ind] = value +notZero(x::AbstractVector) = (!iszero).(x) +Zero(x::AbstractVector) = iszero.(x) +isNan(x::AbstractVector) = isnan.(x) +isInf(x::Number) = abs(x) === Inf +isInf(x::AbstractVector) = isinf.(x) +isNotEqual(x::Number, target::Number) = isequal(isequal(x, target), 0) +isBetween(x, lowerlimit, upperlimit) = lowerlimit < x < upperlimit ? true : false +absolute(x::AbstractVector) = abs.(x) +vecEleMul(x::AbstractVector, y::AbstractVector) = x .* y +vecEleMul(x::Number, y::AbstractVector) = x .* y +expDecay(initialValue::Number, decayFactor::Number, timePass::Number) = + initialValue * (1 - decayFactor)^timePass +mul!(x::AbstractVector, y::AbstractVector) = x .*= y +mul(x::AbstractVector, y::AbstractVector) = x .* y +allTrue(args...) = false ∈ [args...] ? false : true + +ReLu(x::Number) = max(0, x) + +updateVector!(x::AbstractVector, target::Number) = x .= target +updateVector!(x::AbstractVector, target::AbstractArray) = x .= target + +function selectAdd!(x::AbstractVector, ind::AbstractVector, value::AbstractVector) + @. x = x + (ind * value) +end + + + +""" FindIndex(input::String, target::Char) + + Arguments: + text, input text + target, target character + Return: + (a bool vector of match/not match, position vector of the matched) + + Example: + ```jldoctest + julia> using GeneralUtils + julia> text = "Hello World!" + julia> findIndex(text, 'l') + (Bool[0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0], [3, 4, 10]) + ``` +""" +function findIndex(text::String, target::Char) + charlist = [i for i in text] + match_position = isequal.(charlist, target) + match_index = findall(isequal.(match_position, 1)) + + return match_position, match_index +end + +function findIndex(input::Array, target::Number) + match_position = isequal.(input, target) + match_index = findall(match_position) + return match_position, match_index +end + +# function findIndex(input::Array, target::Array) +# match_position = isone.(zeros(length(input))) +# for i in target +# match_position = match_position + isequal.(input, i) +# end + +# match_position = replaceMoreThan.(match_position, 1) +# match_index = findall(isone.(match_position)) # Findall donot work with Int64 vector [1, 0, 0, 1]. +# # It only works with BitVector. isone() converts Int64 vector [1, 0, 0, 1] into +# # BitVector [1, 0, 0, 1] + +# return match_position, match_index +# end + +function findIndex(input::Array, target::Symbol) + match_position = isequal.(input, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::String) + match_position = isequal.(collection, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::Array{String}) + match_position = nothing + match_index = nothing + + for i in target + match_pos = isequal.(collection, i) + match_ind = findall(match_pos) + + if match_position === nothing + match_position = match_pos + + else + match_position = hcat(match_position, match_pos) + end + + if match_index === nothing + match_index = match_ind + else + match_index = hcat(match_index, match_ind) + end + end + + return match_position, match_index +end + +function findIndex(collection::OrderedDict, target::Symbol) + collection_keys = keys(collection) + collection_keys_array = [i for i in collection_keys] + match_position = isequal.(collection_keys_array, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findMax(collection::AbstractVector) + maxValue, maxIndex = findmax(collection) + matchPosition = isequal.(collection, maxValue) + return maxValue, maxIndex, matchPosition +end + + + +""" read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + + with multiple text file in a folder, + this function read x_th text file in a folder (filename is sorted by OS) + + # Example + utils.read_textfile_by_index(cleaned_data_path, 2) + read 2nd txt file in a folder +""" +function read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + if isdir(folder_path) + filenumber = length(readdir(folder_path)) + + if read_file_number > filenumber + error("you specified read_file_number = $read_file_number which is out + of range, the cleaned data folder has only $filenumber files") + return nothing, nothing, nothing + else + content = 0 + # open each file in the directory and read + filename = readdir(folder_path, join=true, sort=false)[read_file_number] + f = open(filename) + content = readlines(f) + # content = read(f) + close(f) + end + + return read_file_number, filename, content + else + error("ERROR no file or folder at $folder_path") + return nothing, nothing, nothing + end + +end + +#------------------------------------------------------------------------------------------------100 + +""" Array_to_JSON3_str(data::AbstractArray) + + encode Array to JSON3 String + + # Example + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] + json3_str = Array_to_JSON3_str(a) + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} +""" +function Array_to_JSON3_str(data::AbstractArray) + d = Dict("Array"=> data, "size"=>size(data)) + json3_str = JSON3.write(d) + return json3_str +end + +#------------------------------------------------------------------------------------------------100 + +""" JSON3_str_to_Array(json3_str::String) + + decode JSON3 String to Array + + # Example + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} + a = JSON3_str_to_Array(json3_str) + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] +""" +function JSON3_str_to_Array(json3_str::String) + d = JSON3.read(json3_str) + array = reshape(Array(d.Array), (d.size[1], d.size[2])) + return array +end + +#------------------------------------------------------------------------------------------------100 + +""" Convert JSON3.read object to OrderedDict + + # Example + dict = dictionary(["a"=>4, "b"=>6]) + OrDict = OrderedDict(dict) + jsonString = JSON3.write(OrDict) # use jsonString to exchange. One can save it to file or send it thru pub/sub + jsonObject = JSON3.read(jsonString) + OrDict2 = JSON3read_to_OrDict(jsonObject) # example here + Adict2 = dictionary(OrDict2) + + Andyferris's github https://github.com/andyferris/Dictionaries.jl +""" +function JSON3read_to_OrDict(x) + dict = OrderedDict() + for (k, v) in x + k = string(k) + dict[k] = v + end + return dict +end + +#------------------------------------------------------------------------------------------------100 + +""" + print time of cpu executtion at the line inwhich this macro is used +""" +macro timeline(expr) + quote + print("line ", $(__source__.line), ": ") + @time $(esc(expr)) + end +end + + +batchindex(batch_counter::Number, batch_size::Number; offset=0) = + (offset + (batch_counter-1) * batch_size + 1) : offset + (batch_counter * batch_size) + +function flip_true_false(x::Bool) + if x == true + x = false + elseif x == false + x = true + else + error("undefined condition line $(@__LINE__)") + end + + return x +end + +function flip_true_false(x::Int) + if x == 1 + x = 0 + elseif x == 0 + x = 1 + else + throw("not define input of type $(typeof(x)) yet") + end + + return x +end + +""" + Return drawed index + # Example + drawed_index = randomWithProb([0.5, 0.2, 0.3]) + +""" +randomWithProb(probability::AbstractVector) = rand(Distributions.Categorical(probability)) # return drawed index + +""" + Draw from choices according to its probability. + Probability range is 0.0 to 1.0 and all probability must summed up to 1 + (may get probability from NNlib's softmax function) + + # Example + + draw = randomChoiceWithProb([true, false, nothing], [0.5, 0.2, 0.3]) +""" +function randomChoiceWithProb(choices::Array, probability::Array) + if length(choices) != length(probability) + error("random is not possible, choices array length != probability array length") + elseif sum(probability) != 1.0 + error("probability does not sum to 1.0") + end + + return choices[randomWithProb(probability)] +end + +function randomChoiceOnTarget(target::Number, targetMatch::Number, choices::AbstractVector, + probability::AbstractVector) + if length(choices) != length(probability) + throw("random is not possible, choices array length != probability array length") + end + return target == targetMatch ? randomChoiceWithProb(choices, probability) : target + # dist = Distributions.Categorical(probability) + # draw_result = choices[rand(dist)] +end + +function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector, + probability::AbstractVector) + return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,)) +end + +function linearly_weighted_avg(a::Array) + total = 0.0 + for (i, v) in enumerate(a) + total = total + (i * v) + end + + return total / sum(a) +end + + +""" Convert String that is holded inside a variable to Symbol +# Example +x = "hello" # x is a variable holding String "hello" \n +y = variable_to_symbol(x) # y holds :hello +""" +function variable_str_to_symbol(variable) + semi = :($variable) + symbol = Symbol(semi) + + return symbol +end + + +""" get useable type of specified fieldname inside a composite struct + + # Example + julia> @Base.kwdef mutable struct some_struct + a::Union{Bool, Nothing} = nothing + b::Union{Float64, Nothing} = nothing + c::Union{Int64, AbstractFloat} = 3.5 + d::Union{String, Nothing} = nothing + end + + julia> a = some_struct() + julia> fieldname_useable_type(some_struct, :c) =result=> [Int64, Float64] +""" +function fieldname_useable_type(somestruct, fieldname::Symbol; + test_types=[2.0, 2, true, :h, "str", 'c', missing, nothing])::Vector{DataType} + new_instance = somestruct() + useable_type = [] + + for i in test_types + try + new_instance.:($fieldname) = i + type = typeof(new_instance.:($fieldname)) + if type ∉ useable_type + push!(useable_type, type) + end + catch + + end + end + + return useable_type +end + + +function randomNoRepeat(drawOptions::Array, draw_number::Integer; + exclude_list::Union{AbstractArray,Nothing}=nothing) + draw_option = copy(drawOptions) + draw_option = isnothing(exclude_list) ? draw_option : + filter!(x -> x ∉ exclude_list, draw_option) + shuffle!(draw_option) + drawed_items = [] + while length(drawed_items) < draw_number + push!(drawed_items, pop!(draw_option)) + end + return drawed_items +end + +""" using cron to schedule backup job by + 1. sudo nano /etc/crontab <<< this is a system-wide cron file + 2. to execute julia file @ 2.00am everyday add the following line at the buttom of the file + 0 2 * * * root julia-1.7 /home/syncthing_backup_script.jl + + Requirements using Dates +""" +function folderBackup(sourceFolderAbsolutePath::String, # absolute path to folder to be backuped + backupFolderAbsolutePath::String; # absolute path to folder used to store backup file + totalBackupFiles::Integer=7, # total backup file, the oldest will be deleted + containerName::Union{Array{String}, Nothing}=nothing) # container using source_folder + + sep = (Sys.iswindows() ? "\\" : '/') + + if sourceFolderAbsolutePath[end] == sep + sourceFolderAbsolutePath = sourceFolderAbsolutePath[1:end-1] + end + if backupFolderAbsolutePath[end] != sl + backupFolderAbsolutePath = backupFolderAbsolutePath * sep + end + + if isdir(backupFolderAbsolutePath) + else + mkpath(backupFolderAbsolutePath) + end + + # stop running docker container service + if containerName !== nothing + println("stop running services") + for i in containerName + try run(`docker stop $i`) catch; end + sleep(10) # wait for services to stop + end + end + + # do backup + println("doing backup now") + timestamp = string(Dates.now()) + name = split(sourceFolderAbsolutePath, sep)[end] * "--" + filename = name * timestamp * ".zip" # resulting compressed filename + run(`chmod -R a+rwx $sourceFolderAbsolutePath`) + # zip -r [destination+filename] [source folder to be zipped] + run(`zip -r $(backupFolderAbsolutePath * filename) $sourceFolderAbsolutePath`) + + # check if total backup file is more than user specified, if yes, delete the oldest backup + backupFiles = readdir(backupFolderAbsolutePath) + + while length(backupFiles) > totalBackupFiles + run(`rm $(backupFolderAbsolutePath * backupFiles[1])`) + backupFiles = readdir(backupFolderAbsolutePath) + end + + # start docker services + if containerName !== nothing + println("start services") + for i in containerName + try run(`docker start $i`) catch; end + sleep(10) # wait for services to stop + end + end +end + +function lowerclip!(data::AbstractVector, lowerbound::Number) + replace!(x -> x < lowerbound ? lowerbound : x, data) +end + +function upperclip!(data::AbstractVector, upperbound::Number) + replace!(x -> x > upperbound ? upperbound : x, data) +end + +function normalise(x::AbstractArray, mu, std) + ϵ = oftype(x[1], 1e-5) + μ = mu + # σ = std(x, dims=dims, mean=μ, corrected=false) # use this when Zygote#478 gets merged + σ = std + return (x .- μ) ./ (σ .+ ϵ) +end + +function minMaxScaler(x::AbstractVector) + min = findmin(x)[1] + max = findmax(x)[1] + + scaler(a::Number, min::Number, max::Number) = (a-min) / (max-min) + return scaler.(x, min, max) +end + +""" a = [-1e200, -1e-200, 1e200, 1e-200] \n + result = vtclamp.(a, 1e-6, 1e6, -1e6, -1e-6) +""" +function customclamp(x::Number, poslo::Number, poshi::Number, + neglo::Number, neghi::Number) + signx = sign(x) + if signx == -1 + if neghi < x < 0 + return neghi + elseif x < neglo + return neglo + else + return x + end + elseif signx == +1 + if poshi < x + return poshi + elseif 0 < x < poslo + return poslo + else + return x + end + end +end + +function unitVec(x::AbstractVector) + y = √(sum(x.^2)) + return x./y +end + +function replaceAt!(x::AbstractVector, ind::AbstractVector, value::Number) + for i in ind + x[i] = value + end +end + +function signbitVec(x::AbstractVector) + sign = signbit.(x) * 1 + signVec = replace(s -> s == 0 ? -1 : s, sign) + return signVec +end + +function deleteall!(x::AbstractVector) + for i in 1:length(x) + deleteat!(x, 1) + end +end + +""" Select specific range of vectors in a dict, return a new dict +# Example +dict = Dict(:a => [1:5...], + :b => [6:10...]) + +call -> selectRange(dict, 1:3) +return -> Dict{Any, Any} with 2 entries: + :a => [1, 2, 3] + :b => [6, 7, 8] +""" +function selectRange(d::Dict{Symbol, <:AbstractVector}, range) + newDict = Dict{Symbol, AbstractVector}() + for (k, v) in d + newDict[k] = v[range] + end + + return newDict +end + +""" Assign value to a given Dict by array of keys + + # Example + d = Dict( + :a1=> Dict(:c=> 5), + :a2=> Dict( + :k=> 10, + :b=> Dict( + :s=> "target", + ) + ) + ) + index = [:a2, :b, :s] \n + assignDict!(d, [:a2, :b, :s], "wow") + + return 1 if no target key in a given dict. +""" +function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign) + wd = nothing + for i in accessArray + println(i) + if i != accessArray[end] + if wd === nothing && haskey(dict, i) + wd = Ref(dict[i]) + elseif wd.x !== nothing && haskey(wd.x, i) + wd = Ref(wd.x[i]) + else + return 1 # error, no target key in a given dict. + end + else + wd.x[i] = valueToAssign + return 0 + end + end +end + +""" convert hour(0-23), minute(0-59) into julia time object +# Example + time +""" +function iTime(h::Integer, m::Integer) + if h == 0 + h = 12 + ampm = "am" + elseif 1 <= h <= 11 + ampm = "am" + elseif h == 12 + ampm = "pm" + elseif 13 <= h <= 23 + h = h - 12 + ampm = "pm" + else + error("hour out of range") + end + + m = m < 10 ? "0$m" : m + t = "$h:$m$ampm" + + return Time(t, "HH:MMp") +end + +""" replace a number according to the limit + if value is lower than lowerbound return lowerbound replacement value + if value is more than upperbound return upperbound replacement value + + # Example + limitvalue(4, (-5 => 0), (5 => 5)) +""" +function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair) + lwLimit, lwReplace = lowerbound + upLimit, upReplace = upperbound + + if v < lwLimit + v = lwReplace + elseif v > upLimit + v = upReplace + else + end + return v +end + +""" assign matrix b to matrix a according to matrix b's CartesianIndex +""" +function cartesianAssign!(a, b) + for (i, v) in enumerate(b) + a[CartesianIndices(b)[i].I...] = v + end + return nothing +end +cartesianAssign!(a::CuArray, b::CuArray) = @cuda cartesianAssign!(a, b) + +function sumAlongDim3(a::Array) + totalDim = length(size(a)) + + if totalDim == 3 + d1, d2, d3 = size(a) + r = zeros(1, 1, d3) + for i in 1:d3 + view(r, 1, 1, i) .= sum(a[:, :, i]) + end + elseif totalDim == 4 + d1, d2, d3, d4 = size(a) + r = zeros(1, 1, d3, d4) + for j in 1:d4 + for i in 1:d3 + view(r, 1, 1, i, j) .= sum(a[:, :, i, j]) + end + end + else + error("this condition is not define yet") + end + + return r +end + + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all slice of 3D another matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> input = rand(32, 32, 128) # batch at 3rd dim + julia> weight = rand(32, 32, 1024) + julia> r = matMul_3Dto3D_manyTo1batch(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul_3Dto3D_manyTo1batch(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if resultStorage === nothing + resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) + end + + c = [slice .* b for slice in eachslice(a, dims=3)] + resultStorage .= cat(c..., dims=4) + + return resultStorage +end + +# function matMul_3Dto3D_manyTo1batch(a::CuArray, b::CuArray; # XXX working code +# resultStorage::Union{CuArray, Nothing}=nothing, threads=256) +# asize = [size(a)...] +# bsize = [size(b)...] +# if resultStorage === nothing +# resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) |> gpu +# end +# CUDA.@sync begin +# @cuda threads=threads matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) +# end +# return resultStorage +# end + +# function matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage) # XXX working code +# _, _, _, p = size(resultStorage) + +# index = threadIdx().x # this example only requires linear indexing, so just use `x` +# stride = blockDim().x +# for i in index:stride:p +# view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b +# end +# return nothing +# end + +""" GPU kernel +""" +function matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage, linear_to_cartesian) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + if i <= size(a, 3) # guard against unused threads to accessing memory out of bound + cartesianIndex = linear_to_cartesian(i, size(b)) # example for how to send "inner" function to gpu + # @cuprintln("gpu thread $i $cartesianIndex[2]") + + @. @views resultStorage[:, :, :, i] = a[ :, :, i] * b + # view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b # alternative code + # @view(resultStorage[:, :, :, i]) .= @view(a[ :, :, i]) .* b # alternative code + end + return nothing +end + +""" GPU version of batchMatEleMul + + Example + julia> using Flux, CUDA + julia> device = Flux.CUDA.functional() ? gpu : cpu + julia> if device == gpu CUDA.device!(0) end + julia> input = rand(32, 32, 128) |> gpu; # 128-batches + julia> weight = rand(32, 32, 1024) |> gpu; # 1-batch. this matrix is essentially (32, 32, 1024, 1) + julia> r = matMul_3Dto3D_manyTo1batch(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul_3Dto3D_manyTo1batch(a::CuArray, b::CuArray; + resultStorage::Union{CuArray, Nothing}=nothing) + if resultStorage === nothing + resultStorage = similar(a, eltype(b), size(b, 1), size(b, 2), size(b, 3), size(a, 3)) |> gpu + end + + kernel = @cuda launch=false matMul_3Dto3D_manyTo1batch_gpu!(a, b, resultStorage, GeneralUtils.linear_to_cartesian) + config = launch_configuration(kernel.fun) + + # threads to be launched. Since one can't launch exact thread number the kernel needs, + # one just launch threads more than this kernel needs then use a guard inside the kernel + # to prevent unused threads to access memory. + threads = min(256, config.threads) # threads per block. depend on gpu. Most NVIDIA gpu has 1024 threads per block + + # total desired threads to launch to gpu. Usually 1 thread per 1 matrix element + totalThreads = size(a, 3) # This kernel use 1 thread per batch + + blocks = cld(totalThreads, threads) + + CUDA.@sync begin + kernel(a, b, resultStorage, GeneralUtils.linear_to_cartesian; threads, blocks) + end + return resultStorage +end + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all batch of another 4D matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> + julia> a = rand(2,2,3) # 3-batches + julia> b = rand(2,2,4,3) # 3-batches + julia> r = GeneralUtils.matMul_3Dto4D_batchwise(a, b); + julia> size(r) + (2, 2, 4, 3) +""" +function matMul_3Dto4D_batchwise(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + if size(a, 3) != size(b, 4) + error("batch number of a and b must be equal") + end + if resultStorage === nothing + resultStorage = zeros(size(b, 1), size(b, 2), size(b, 3), size(a, 3)) + end + for i in 1:size(a, 3) + view(resultStorage, :, :, :, i) .= a[:, :, i] .* b[:, :, :, i] + end + return resultStorage +end + +""" GPU-compatible linear index to cartesian index conversion +""" +function linear_to_cartesian(i::Int, arraySize::NTuple{4,Int}) + # Check that the linear coordinate is valid + # prod(arraySize) is the same as *(arraySize...). they multipy all elements in an array. + # but this code use prod() because splat breaks GPU performance + if i < 1 || i > prod(arraySize) + error("Invalid linear coordinate") + end + # Extract the dimensions of the matrix + n1, n2, n3, n4 = arraySize + # Compute the cartesian coordinate using rem and div functions + i1 = ((i-1) % (n1)) + 1 # +1 convert 0-based to 1-based index + i2 = ((i-1) ÷ (n1)) % n2 + 1 + i3 = ((i-1) ÷ (n1*n2)) % n3 + 1 + i4 = (i-1) ÷ (n1*n2*n3) + 1 + # Return the cartesian coordinate as a tuple + return (i1, i2, i3, i4) +end + +""" return a vector with true at max value and false for other value. + if vector is all-zeros, return all-false vector. +""" +function vectorMax(x) + if sum(isNotEqual.(x, 0)) == 0 # guard against all-zeros array + # instead of returning all-zeros original vector, + # return all-false vector to prevent type instability + return isNotEqual.(x, 0) + else + return isequal.(x, maximum(x)) + end +end + +function multiply_last(matrix, x, n) + # X is the scalar to multiply + # matrix is the column-major 2D matrix + # n is the number of elements to be multiplied, starting from the last one + # returns a new matrix with the same shape as the original one + + # get the number of rows and columns of the matrix + rows, cols = size(matrix) + + # create a copy of the matrix to avoid mutating the original one + result = copy(matrix) + + # loop over the last n elements in column-major order + for i in (rows * cols - n + 1):(rows * cols) + # get the row and column indices of the current element + row = (i - 1) % rows + 1 + col = (i - 1) ÷ rows + 1 + + # multiply the element by X and store it in the result matrix + result[row, col] *= x + end + + # return the result matrix + return result +end + +function multiply_random_elements(A, x, n, rng=MersenneTwister(1234)) + # rng is a random number generator object, see https://docs.julialang.org/en/v1/stdlib/Random/ + # x is a scalar value to multiply by + # A is a column-major 2D matrix or a vector + # n is the number of elements to be multiplied + # returns a new array with n randomly chosen distinct elements multiplied by x + B = copy(A) # make a copy of A to avoid mutating it + d = ndims(A) # get the number of dimensions of A + if d == 1 # if A is a vector + m = length(A) # get the length of A + indices = collect(1:m) # create an array of indices from 1 to m + shuffle!(rng, indices) # shuffle the indices in-place using the RNG + for i in 1:n # loop n times + j = indices[i] # get the i-th shuffled index + B[j] *= x # multiply the element at j by x + end + elseif d == 2 # if A is a matrix + m = size(A, 1) # number of rows in A + p = size(A, 2) # number of columns in A + indices = collect(1:m*p) # create an array of linear indices from 1 to m*p + shuffle!(rng, indices) # shuffle the indices in-place using the RNG + for i in 1:n # loop n times + j = indices[i] # get the i-th shuffled index + B[j] *= x # multiply the element at j by x + end + else # if A is neither a vector nor a matrix + error("A must be a vector or a matrix") + end + return B # return the new array +end + +""" Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and + replace matrix A's elements of the same position with value, a. + + Example + julia> mask = rand([-1,0,1],4,4,1) + julia> A = rand(4,4,1) + julia> C = replaceElements(mask, A, -1, 5.0, 3) +""" +function replaceElements(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number, + n::Int=0; rng::AbstractRNG=MersenneTwister(1234)) + """ Prompt + Write a julia function to operate on column-major 3D matrix. The function randomly + choose elements in matrix mask that has value markValue and replace elements in matrix A at + the same position with value a. The choosing randomness is controlled by rng function. + I also want to specify how many elements to be replaced. + """ + + total_x_tobeReplced = sum(isequal.(mask, markValue)) + if n == 0 || n > total_x_tobeReplced + n = total_x_tobeReplced + end + + # check if mask and A have the same size + if size(mask) != size(A) + error("mask and A must have the same size") + end + C = copy(A) + # get the indices of elements in mask that equal markValue + indices = findall(x -> x == markValue, mask) + # shuffle the indices using the rng function + shuffle!(rng, indices) + # select the first n indices + selected = indices[1:n] + # replace the elements in A at the selected positions with a + for i in selected + C[i] = a + end + return C +end + +""" Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and + replace matrix A's elements of the same position with value, a. if n == 0, all marked value is replaced + + Example + julia> mask = rand([-1,0,1],4,4,1) + julia> A = rand(4,4,1) + julia> C = replaceElements(mask, A, -1, 5.0, 3) +""" +function replaceElements!(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number, + n::Int=0; rng::AbstractRNG=MersenneTwister(1234)) + total_x_tobeReplced = sum(isequal.(mask, markValue)) + remaining = 0 + if n == 0 || n > total_x_tobeReplced + remaining = n - total_x_tobeReplced + n = total_x_tobeReplced + end + + # check if mask and A have the same size + if size(mask) != size(A) + error("mask and A must have the same size, mask $(size(mask)) A $(size(A))") + end + # get the indices of elements in mask that equal markValue + indices = findall(x -> x == markValue, mask) + # shuffle the indices using the rng function + shuffle!(rng, indices) + # select the first n indices + selected = indices[1:n] + # replace the elements in A at the selected positions with a + for i in selected + A[i] = a + end + + return remaining +end + +""" Replace n elements that has value x with user specified value a. +""" +function replaceElements(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234)) + total_x_tobeReplced = sum(isequal.(A, x)) + if n == 0 || n > total_x_tobeReplced + n = total_x_tobeReplced + end + + B = copy(A) + # A is a column-major 3D matrix + # x is the value to be replaced + # a is the new value + # rng is a random number generator function + # n is the number of elements to be replaced + + # find the indices of elements in A that equal x + indices = findall(==(x), B) + + # shuffle the indices using the rng function + shuffle!(rng, indices) + + # select the first n indices + selected = indices[1:n] + + # replace the elements at the selected indices with a + for i in selected + B[i] = a + end + + # return the modified matrix A + return B +end + + +function replaceElements!(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234)) + total_x_tobeReplced = sum(isequal.(A, x)) + if n == 0 || n > total_x_tobeReplced + n = total_x_tobeReplced + end + + # A is a column-major 3D matrix + # x is the value to be replaced + # a is the new value + # rng is a random number generator function + # n is the number of elements to be replaced + + # find the indices of elements in A that equal x + indices = findall(==(x), A) + + # shuffle the indices using the rng function + shuffle!(rng, indices) + + # select the first n indices + selected = indices[1:n] + + # replace the elements at the selected indices with a + for i in selected + A[i] = a + end +end + + + + + +""" Get characters between specified characters. + + Arguments: + startChar, start character + endChar, end character. + endCharPosition, end character position after startChar. Can be "next" or "end". + "next" means the closed endChar just after startChar. + "end" means the furthest endChar. + includeChar, whether to include the startChar and endChar. Default is true + Return: + the characters between specified characters. + + Example: + ```jldoctest + julia> using GeneralUtils + julia> text = "{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n }}\n \n " + julia> getStringBetweenCharacters(text, '{', '}', endCharPosition="end") + {\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n }} + ``` +""" +function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char; + endCharLocation::String="next", includeChar::Bool=true) where {T<:AbstractString} + + # get the position of the startChar + startCharPosition = findfirst(startChar, text) + endCharPosition = nothing + + if endCharLocation == "end" + # get the first position of the endChar coming from the end of text + endCharPosition = findlast(endChar, text) + elseif endCharLocation == "next" + # get the first position of the endChar after startCharPosition + endCharPosition = findnext(endChar, text, startCharPosition + 1) + else + error("endCharPositio must be \"end\" or \"next\"") + end + + @show startCharPosition, endCharPosition + + # get characters between startChar and endChar from text + extractedText = text[startCharPosition:endCharPosition] + # convert substring to string + extractedText = string(extractedText) + + extractedText = includeChar == true ? extractedText : extractedText[2:end-1] + + return extractedText +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.3/test/etc.jl b/previousVersion/0.0.3/test/etc.jl new file mode 100644 index 0000000..e69de29 diff --git a/src/GeneralUtils.jl b/src/GeneralUtils.jl new file mode 100644 index 0000000..ec47233 --- /dev/null +++ b/src/GeneralUtils.jl @@ -0,0 +1,89 @@ +module GeneralUtils + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue + +include("util.jl") +using .util + +include("communication.jl") +using .communication + +include("llmUtil.jl") +using .llmUtil + +include("interface.jl") +using .interface + + +#------------------------------------------------------------------------------------------------100 + +""" version 0.0.4 + Todo: + - [*1] cartesianAssign for different matrix dimension + + Change from version: 0.0.3 + - + + All features + +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # generalUtils \ No newline at end of file diff --git a/src/communication.jl b/src/communication.jl new file mode 100644 index 0000000..12e10f3 --- /dev/null +++ b/src/communication.jl @@ -0,0 +1,640 @@ +module communication + +export generate_msgMeta, isMqttConnectionAlive, checkMqttConnection!, + sendMqttMsg, sendReceiveMqttMsg, mqttClientInstance + +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient +using ..util + +# ---------------------------------------------- 100 --------------------------------------------- # + +mutable struct mqttClientInstance + mqttBrokerAddress::String + mqttBrokerPort::Integer + subtopic::Vector{String} + receiveDataChannel::Channel + onMsgCallback::Function + qos::MQTTClient.QOS + client::MQTTClient.Client + connection::MQTTClient.Connection + keepalivetopic::String + keepaliveChannel::Channel + keepaliveCheckInterval::Integer # second + lastTimeMqttConnCheck::DateTime +end + +function mqttClientInstance( + mqttBrokerAddress::String, + mqttBrokerPort::Integer, + subtopic::Vector{String}, + receiveDataChannel::Channel, + keepaliveChannel::Channel, # user needs to specify because it has to be accessible by user-defined onMsgCallback() + onMsgCallback::Function + ; + keepalivetopic::String= "/keepalive/$(uuid4())", + keepaliveCheckInterval::Integer=30, + qos::MQTTClient.QOS=QOS_1, + ) + + client, connection = MQTTClient.MakeConnection(mqttBrokerAddress, mqttBrokerPort) + MQTTClient.connect(client, connection) + for i in subtopic + MQTTClient.subscribe(client, i, onMsgCallback, qos=qos) + end + MQTTClient.subscribe(client, keepalivetopic, onMsgCallback, qos=qos) + + instance = mqttClientInstance( + mqttBrokerAddress, + mqttBrokerPort, + subtopic, + receiveDataChannel, + onMsgCallback, + qos, + client, + connection, + keepalivetopic, + keepaliveChannel, + keepaliveCheckInterval, + Dates.now(), + ) + + return instance +end + + +""" Generate msgMeta to be including in a message. So the message receiver know + what to do with the message. + + Arguments\n + ----- + sendTopic::String + topic the sender sends to e.g. "/agent/wine/api/v1/prompt" + + Keyword Arguments\n + ----- + msgPurpose::String + purpose of this message e.g. "updateStatus" + senderName::String + sender name (String) e.g. "agent-wine-web-frontend" + senderId::String + sender id e.g. string(uuid4()) + receiverName::String + msg receiver name (String) e.g. "agent-backend" + receiverId::String + msg receiver id, nothing means everyone in the topic e.g. string(uuid4()) + requestresponse::String + this message is a "request" or "response" + getpost::String + this message is a "get" or "post" + msgId::String + message ID e.g. string(uuid4()) + timestamp::String + message published timestamp e.g. string(Dates.now()) + senderselfnote::Any + help sender manage incoming reply msg, could be "for text inference", "img generation" + replyTopic::String + sender ask receiver to reply to this topic e.g. "/agent/frontend" + replyToMsgId::String + sender is responding to this msg ID e.g. string(uuid4()) + acknowledgestatus::String + e.g. "acknowledged" + mqttBrokerAddress::String + MQTT broker URI. e.g. "test.mosquitto.org" + mqttBrokerPort::Integer + MQTT broker port + + Return\n + ----- + msgMeta::Dict{Symbol, Any} + + Example\n + ----- + ```jldoctest + julia> using Revise + julia> using GeneralUtils + julia> msgMeta = GeneralUtils.generate_msgMeta( + "/agent/frontend/wine/chat/api/v1/txt/receive", + msgPurpose="keepalive", + senderName="keepaliveservice") + ``` + + Signature\n + ----- +""" +function generate_msgMeta( + sendTopic::T1, # topic the sender sends to e.g. "/agent/wine/api/v1/prompt" + ; + msgPurpose::Union{T1, Nothing}= nothing, # purpose of this message e.g. "updateStatus" + senderName::Union{T1, Nothing}= nothing, # sender name (String) e.g. "agent-wine-web-frontend" + senderId::Union{T1, Nothing}= nothing, # sender id e.g. string(uuid4()) + receiverName::Union{T1, Nothing}= nothing, # msg receiver name (String) e.g. "agent-backend" + receiverId::Union{T1, Nothing}= nothing, # id of msg receiver, nothing means everyone in the topic e.g. string(uuid4()) + requestresponse::Union{T1, Nothing}= nothing, # this message is a "request" or "response" + getpost::Union{T1, Nothing}= nothing, # this message is a "get" or "post" + msgId::Union{T1, Nothing}= string(uuid4()), + timestamp= string(Dates.now()), # message published timestamp + + # help sender manage incoming reply msg, could be "for text inference", "img generation" + senderselfnote::Any= nothing, + + # sender ask receiver to reply to this topic + # e.g. "/agent/frontend/wine/chat/api/v1/txt/receive" + replyTopic::Union{T1, Nothing}= nothing, + + replyToMsgId::Union{T1, Nothing}= nothing, # sender is responding to this msg ID + acknowledgestatus::Union{T1, Nothing}= nothing, # "acknowledged", + + mqttBrokerAddress::T1= "test.mosquitto.org", + mqttBrokerPort::Integer= 1883, + + msgFormatVersion::Union{T1, Nothing}= nothing, + + )::Dict{Symbol, Any} where {T1<:AbstractString} + + msgMeta::Dict=Dict( + :sendTopic=> sendTopic, # topic the sender sends to e.g. "/agent/wine/api/v1/prompt" + :msgPurpose=> msgPurpose, # purpose of this message e.g. "updateStatus" + :senderName=> senderName, # sender name (String) e.g. "agent-wine-web-frontend" + :senderId=> senderId, # sender id e.g. string(uuid4()) + :receiverName=> receiverName, # msg receiver name (String) e.g. "agent-backend" + :receiverId=> receiverId, # msg receiver id, nothing means everyone in the topic e.g. string(uuid4()) + :requestResponse=> requestresponse, # this message is a "request" or "response" + :getPost=> getpost, # this message is a "get" or "post" + :msgId=> msgId, + :timeStamp=> timestamp, # message published timestamp + + # help sender manage incoming reply msg, could be "for text inference", "img generation" + :senderSelfnote=>senderselfnote, + + # sender ask receiver to reply to this topic + # e.g. "/agent/frontend/wine/chat/api/v1/txt/receive" + :replyTopic=> replyTopic, + + :replyToMsgId=> replyToMsgId, + :acknowledgestatus=> acknowledgestatus, + + :mqttBrokerAddress=> mqttBrokerAddress, + :mqttBrokerPort=> mqttBrokerPort, + + :msgFormatVersion=> msgFormatVersion, + ) + + return msgMeta +end + + + +""" Check mqtt server connection. + + Arguments\n + ----- + mqttInstanceDict::Dict{Symbol, Any} + a dictionary contain mqtt instance. 1 per mqtt client. + interval::Integer + time interval to check mqtt server in seconds + + Return\n + ----- + isconnectionalive::Bool + true if mqtt connection is alive + + Example\n + ----- + ```jldoctest + julia> using Revise + julia> using GeneralUtils, Dates, JSON3, MQTTClient + julia> mqttMsgReceiveChannel = Channel(8) + julia> keepaliveChannel = Channel(8) + julia> function onMsgCallback(topic, payload) + jobj = JSON3.read(String(payload)) + onMsg = copy(jobj) + put!(mqttMsgReceiveChannel, onMsg) + end + julia> mqttInstanceDict = Dict{Symbol, Any}( + :mqttServerInfo=> Dict( + :broker=> "test.mosquitto.org", + :port=> 1883, + ), + :onMsgCallback=> onMsgCallback, + :client=> nothing, + :connection=> nothing, + :subtopic=> ["/servicetopic", "/keepalivetopic"], + :keepalivetopic=> "/keepalive", + :keepaliveChannel=> keepaliveChannel, + :lastTimeMqttConnCheck=> Dates.now(), + ) + julia> mqttInstanceDict[:client], mqttInstanceDict[:connection] = + MakeConnection(mqttInstanceDict[:mqttServerInfo][:broker], + mqttInstanceDict[:mqttServerInfo][:port]) + julia> connect(mqttInstanceDict[:client], mqttInstanceDict[:connection]) + julia> mqttConnStatus = GeneralUtils.isMqttConnectionAlive(mqttInstanceDict) + julia> println(mqttConnStatus) + ``` + + Signature\n + ----- +""" +function isMqttConnectionAlive(mqttInstanceDict::Dict{Symbol, Any})::Bool + + starttime = Dates.now() + isconnectionalive = false + + # ditch old keepalive msg is any + while isready(mqttInstanceDict[:keepaliveChannel]) + _ = take!(mqttInstanceDict[:keepaliveChannel]) + end + + msgMeta = generate_msgMeta( + mqttInstanceDict[:keepalivetopic], + msgPurpose= "keepalive", + ) + + keepaliveMsg = Dict( + :msgMeta => msgMeta, + :text=>"keepalive", + ) + + publish(mqttInstanceDict[:client], keepaliveMsg[:msgMeta][:sendTopic], + JSON3.write(keepaliveMsg)) + + timediff = 0 + while timediff < 5 + timediff = timedifference(starttime, Dates.now(), "seconds") + if isready(mqttInstanceDict[:keepaliveChannel]) + incomingMsg = take!(mqttInstanceDict[:keepaliveChannel]) + if incomingMsg[:msgMeta][:msgId] == keepaliveMsg[:msgMeta][:msgId] + # connection is alive + isconnectionalive = true + break + end + end + sleep(1) + end + return isconnectionalive +end +function isMqttConnectionAlive(mqttInstance::mqttClientInstance)::Bool + + starttime = Dates.now() + isconnectionalive = false + + # ditch old keepalive msg is any + while isready(mqttInstance.keepaliveChannel) + _ = take!(mqttInstance.keepaliveChannel) + end + + msgMeta = generate_msgMeta( + mqttInstance.keepalivetopic, + msgPurpose= "keepalive", + ) + + keepaliveMsg = Dict( + :msgMeta => msgMeta, + :text=>"keepalive", + ) + + publish(mqttInstance.client, keepaliveMsg[:msgMeta][:sendTopic], + JSON3.write(keepaliveMsg)) + + timediff = 0 + while timediff < 5 + timediff = timedifference(starttime, Dates.now(), "seconds") + if isready(mqttInstance.keepaliveChannel) + incomingMsg = take!(mqttInstance.keepaliveChannel) + if incomingMsg[:msgMeta][:msgId] == keepaliveMsg[:msgMeta][:msgId] + # connection is alive + isconnectionalive = true + break + end + end + sleep(1) + end + return isconnectionalive +end + + +""" Check mqtt server connection, reconnect if disconnected. + + Arguments\n + ----- + mqttInstanceDict::Dict{Symbol, Any} + a dictionary contain mqtt instance. 1 per mqtt client. + interval::Integer + time interval to check mqtt server in seconds + + Return\n + ----- + isreconnect::Bool + true if mqtt connection is reconnected + + Example\n + ----- + ```jldoctest + julia> using Revise + julia> using GeneralUtils, Dates, JSON3, MQTTClient + julia> mqttMsgReceiveChannel = Channel(8) + julia> keepaliveChannel = Channel(8) + julia> function onMsgCallback(topic, payload) + jobj = JSON3.read(String(payload)) + onMsg = copy(jobj) + put!(mqttMsgReceiveChannel, onMsg) + end + julia> mqttInstanceDict = Dict{Symbol, Any}( + :mqttServerInfo=> Dict( + :broker=> "test.mosquitto.org", + :port=> 1883, + ), + :onMsgCallback=> onMsgCallback, # function + :client=> nothing, + :connection=> nothing, + :subtopic=> ["/servicetopic", "/sometopic"], # Vector{String} + :keepalivetopic=> "/keepalive", + :keepaliveChannel=> keepaliveChannel, # Channel{Dict} + :lastTimeMqttConnCheck=> Dates.now(), + ) + julia> mqttInstanceDict[:client], mqttInstanceDict[:connection] = + MakeConnection(mqttInstanceDict[:mqttServerInfo][:broker], + mqttInstanceDict[:mqttServerInfo][:port]) + julia> connect(mqttInstanceDict[:client], mqttInstanceDict[:connection]) + julia> GeneralUtils.checkMqttConnection!(mqttInstanceDict, 5) + ``` + + Signature\n + ----- +""" +function checkMqttConnection!(mqttInstanceDict::Dict{Symbol, Any}, interval::Integer)::Bool + isreconnect = false + + # check if mqtt connection is still up + intervaldiff = timedifference(mqttInstanceDict[:lastTimeMqttConnCheck], Dates.now(), "seconds") + + if intervaldiff > interval + while true + mqttConnStatus = isMqttConnectionAlive(mqttInstanceDict) + if mqttConnStatus == false + println("Attemping to reconnect") + # use new client to reconnect instead of the previous one because I don't want to modify MQTTClient.jl yet + mqttInstanceDict[:client], mqttInstanceDict[:connection] = + MakeConnection(mqttInstanceDict[:mqttServerInfo][:broker], + mqttInstanceDict[:mqttServerInfo][:port]) + connect(mqttInstanceDict[:client], mqttInstanceDict[:connection]) + for topic in mqttInstanceDict[:subtopic] + subscribe(mqttInstanceDict[:client], topic, mqttInstanceDict[:onMsgCallback], qos=QOS_1) + end + isreconnect = true + println("reconnected") + else + mqttInstanceDict[:lastTimeMqttConnCheck] = Dates.now() + break + end + end + end + + return isreconnect +end +function checkMqttConnection!(mqttInstance::mqttClientInstance; + keepaliveCheckInterval::Union{Integer, Nothing}=nothing)::Bool + + interval = keepaliveCheckInterval !== nothing ? keepaliveCheckInterval : mqttInstance.keepaliveCheckInterval + isreconnect = false + + # check if mqtt connection is still up + intervaldiff = timedifference(mqttInstance.lastTimeMqttConnCheck, Dates.now(), "seconds") + + if intervaldiff > interval + while true + mqttConnStatus = isMqttConnectionAlive(mqttInstance) + if mqttConnStatus == false + println("Attemping to reconnect") + # use new client to reconnect instead of the previous one because I don't want to modify MQTTClient.jl yet + mqttInstance.client, mqttInstance.connection = + MakeConnection(mqttInstance.mqttBrokerAddress, + mqttInstance.mqttBrokerPort) + connect(mqttInstance.client, mqttInstance.connection) + for topic in mqttInstance.subtopic + subscribe(mqttInstance.client, topic, mqttInstance.onMsgCallback, qos=mqttInstance.qos) + end + MQTTClient.subscribe(mqttInstance.client, mqttInstance.keepalivetopic, mqttInstance.onMsgCallback, qos=mqttInstance.qos) + isreconnect = true + println("reconnected") + else + mqttInstance.lastTimeMqttConnCheck = Dates.now() + break + end + end + end + + return isreconnect +end + + +""" Send a message to specified MQTT topic then wait for reply. + +# Arguments + - `outgoingMsg::Dict` + an outgoing message + +# Return + - `result::NamedTuple` + ( isDone= true, error= nothing ) + +# Example + ```jldoctest + julia> using Revise + julia> using GeneralUtils, Dates, JSON3, UUIDs + julia> msgMeta = GeneralUtils.generate_msgMeta( + "/testtopic", + senderName= "somename", + senderId= string(uuid4()), + mqttBrokerAddress= "test.mosquitto.org", + mqttBrokerPort= 1883, + ) + julia> outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict("hello"=> "World"), + ) + julia> isDone, error = GeneralUtils.sendMqttMsg(outgoingMsg) + ``` + +# Signature +""" +function sendMqttMsg(outgoingMsg::Dict{Symbol, T})::NamedTuple where {T<:Any} + try + # Instantiate a client and connection. + client, connection = MakeConnection( + outgoingMsg[:msgMeta][:mqttBrokerAddress], + outgoingMsg[:msgMeta][:mqttBrokerPort]) + connect(client, connection) + + publish(client, outgoingMsg[:msgMeta][:sendTopic], JSON3.write(outgoingMsg)) + # disconnect mqtt + disconnect(client) + return (isDone=true, error=nothing) + catch e + return (isDone=false, error=e) + end +end + +function sendMqttMsg(mqttInstance::mqttClientInstance, outgoingMsg::Dict{Symbol, T} + )::NamedTuple where {T<:Any} + try + publish(mqttInstance.client, outgoingMsg[:msgMeta][:sendTopic], JSON3.write(outgoingMsg)) + # disconnect mqtt + disconnect(client) + return (isDone=true, error=nothing) + catch e + return (isDone=false, error=e) + end +end + + +""" Send a message to specified MQTT topic then wait for reply. + +# Arguments + - `outgoingMsg::Dict{Symbol, T}` + an outgoing message + +# Keyword Arguments + - `timeout::Integer` + time to wait for a response before error + +# Return + - `result::NamedTuple` + ( + isDone= true, # idicates whether sending MQTT message successful + error= nothing # error message + response= somemessage # response message + ) + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils, Dates, UUIDs +julia> msgMeta = GeneralUtils.generate_msgMeta( + "/testtopic", + senderName= "somename", + senderId= string(uuid4()), + mqttBrokerAddress= "mqtt.yiem.cc", + mqttBrokerPort= 1883, + ) +julia> outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict(:hello=> "World"), + ) +julia> isDone, error, response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) +``` + +# Signature +""" +function sendReceiveMqttMsg(outgoingMsg::Dict{Symbol, T}; + timeout::Integer=60, maxattempt::Integer=1)::NamedTuple where {T<:Any} + + mqttMsgReceiveTopic = "/GeneralUtils_sendReceiveMqttMsg/$(outgoingMsg[:msgMeta][:senderId])" + mqttMsgReceiveChannel = Channel(4) + + # ask message receiver to send a message back to specified topic + outgoingMsg[:msgMeta][:replyTopic] = mqttMsgReceiveTopic + + # Define the callback for receiving messages. + function onMsgCallback(topic, payload) + jobj = JSON3.read(String(payload)) + onMsg = copy(jobj) + put!(mqttMsgReceiveChannel, onMsg) + end + + # Instantiate a client and connection. + client, connection = MakeConnection(outgoingMsg[:msgMeta][:mqttBrokerAddress], + outgoingMsg[:msgMeta][:mqttBrokerPort]) + connect(client, connection) + subscribe(client, mqttMsgReceiveTopic, onMsgCallback, qos=QOS_1) + + timepass = nothing + attempts = 1 + while attempts <= maxattempt + publish(client, outgoingMsg[:msgMeta][:sendTopic], JSON3.write(outgoingMsg)) + + starttime = Dates.now() + while true + timepass = timedifference(starttime, Dates.now(), "seconds") + + if timepass <= timeout + if isready(mqttMsgReceiveChannel) + incomingMsg = take!(mqttMsgReceiveChannel) + + if incomingMsg[:msgMeta][:replyToMsgId] == outgoingMsg[:msgMeta][:msgId] + # disconnect mqtt + unsubscribe(client, mqttMsgReceiveTopic) + disconnect(client) + return (isDone=true, error=nothing, response=incomingMsg[:payload]) + end + end + else + break + end + sleep(1) + end + attempts += 1 + println("attempts $attempts ", @__FILE__, " ", @__LINE__) + end + + # disconnect mqtt + unsubscribe(client, mqttMsgReceiveTopic) + disconnect(client) + return (isDone=false, error="no response, timeout $timepass/$timeout, $attempts publish attempted", + response=nothing) +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module communication \ No newline at end of file diff --git a/src/interface.jl b/src/interface.jl new file mode 100644 index 0000000..ec43801 --- /dev/null +++ b/src/interface.jl @@ -0,0 +1,1186 @@ +module interface + + +export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, replaceMoreThan, + replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul3Dto3DmanyTo1batch, + matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, + multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween, + isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!, + getDictPath, dataframeToCSV + +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames, CSV +using ..util, ..communication + +# ---------------------------------------------- 100 --------------------------------------------- # + +noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) +findNotZero(x::AbstractVector) = findall( (!iszero).(x) ) +replaceMoreThan(x, target, replaceValue) = x > target ? replaceValue : x +replaceMoreThan(x, target, a, b) = x > target ? a : b +replaceLessThan(x, target, replaceValue) = x < target ? replaceValue : x +replaceLessThan(x, target, a, b) = x < target ? a : b +replaceBetween(x, lowerbound, upperbound, replaceValue) = lowerbound < x < upperbound ? replaceValue : x +precision(x::Array{<:Array}) = ( std(mean.(x)) / mean(mean.(x)) ) * 100 +precision(x::Array) = std(x) / mean(x) * 100 +replaceAt!(x::AbstractVector, ind::Number, value::Number) = x[ind] = value +notZero(x::AbstractVector) = (!iszero).(x) +Zero(x::AbstractVector) = iszero.(x) +isNan(x::AbstractVector) = isnan.(x) +isInf(x::Number) = abs(x) === Inf +isInf(x::AbstractVector) = isinf.(x) +isNotEqual(x::Number, target::Number) = isequal(isequal(x, target), 0) +isBetween(x, lowerlimit, upperlimit) = lowerlimit < x < upperlimit ? true : false +absolute(x::AbstractVector) = abs.(x) +vecEleMul(x::AbstractVector, y::AbstractVector) = x .* y +vecEleMul(x::Number, y::AbstractVector) = x .* y +expDecay(initialValue::Number, decayFactor::Number, timePass::Number) = + initialValue * (1 - decayFactor)^timePass +mul!(x::AbstractVector, y::AbstractVector) = x .*= y +mul(x::AbstractVector, y::AbstractVector) = x .* y +allTrue(args...) = false ∈ [args...] ? false : true + +ReLu(x::Number) = max(0, x) + +updateVector!(x::AbstractVector, target::Number) = x .= target +updateVector!(x::AbstractVector, target::AbstractArray) = x .= target + +function selectAdd!(x::AbstractVector, ind::AbstractVector, value::AbstractVector) + @. x = x + (ind * value) +end + + + +""" FindIndex(input::String, target::Char) + + Arguments: + text, input text + target, target character + Return: + (a bool vector of match/not match, position vector of the matched) + + Example: + ```jldoctest + julia> using GeneralUtils + julia> text = "Hello World!" + julia> findIndex(text, 'l') + (Bool[0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0], [3, 4, 10]) + ``` +""" +function findIndex(text::String, target::Char) + charlist = [i for i in text] + match_position = isequal.(charlist, target) + match_index = findall(isequal.(match_position, 1)) + + return match_position, match_index +end + +function findIndex(input::Array, target::Number) + match_position = isequal.(input, target) + match_index = findall(match_position) + return match_position, match_index +end + +# function findIndex(input::Array, target::Array) +# match_position = isone.(zeros(length(input))) +# for i in target +# match_position = match_position + isequal.(input, i) +# end + +# match_position = replaceMoreThan.(match_position, 1) +# match_index = findall(isone.(match_position)) # Findall donot work with Int64 vector [1, 0, 0, 1]. +# # It only works with BitVector. isone() converts Int64 vector [1, 0, 0, 1] into +# # BitVector [1, 0, 0, 1] + +# return match_position, match_index +# end + +function findIndex(input::Array, target::Symbol) + match_position = isequal.(input, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::String) + match_position = isequal.(collection, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findIndex(collection::Array{String}, target::Array{String}) + match_position = nothing + match_index = nothing + + for i in target + match_pos = isequal.(collection, i) + match_ind = findall(match_pos) + + if match_position === nothing + match_position = match_pos + + else + match_position = hcat(match_position, match_pos) + end + + if match_index === nothing + match_index = match_ind + else + match_index = hcat(match_index, match_ind) + end + end + + return match_position, match_index +end + +function findIndex(collection::OrderedDict, target::Symbol) + collection_keys = keys(collection) + collection_keys_array = [i for i in collection_keys] + match_position = isequal.(collection_keys_array, target) + match_index = findall(match_position) + + return match_position, match_index +end + +function findMax(collection::AbstractVector) + maxValue, maxIndex = findmax(collection) + matchPosition = isequal.(collection, maxValue) + return maxValue, maxIndex, matchPosition +end + + + +""" read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + + with multiple text file in a folder, + this function read x_th text file in a folder (filename is sorted by OS) + + # Example + utils.read_textfile_by_index(cleaned_data_path, 2) + read 2nd txt file in a folder +""" +function read_textfile_by_index(folder_path::String, read_file_number::Integer=1) + if isdir(folder_path) + filenumber = length(readdir(folder_path)) + + if read_file_number > filenumber + error("you specified read_file_number = $read_file_number which is out + of range, the cleaned data folder has only $filenumber files") + return nothing, nothing, nothing + else + content = 0 + # open each file in the directory and read + filename = readdir(folder_path, join=true, sort=false)[read_file_number] + f = open(filename) + content = readlines(f) + # content = read(f) + close(f) + end + + return read_file_number, filename, content + else + error("ERROR no file or folder at $folder_path") + return nothing, nothing, nothing + end + +end + +# ---------------------------------------------- 100 --------------------------------------------- # + +""" Array_to_JSON3_str(data::AbstractArray) + + encode Array to JSON3 String + + # Example + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] + json3_str = Array_to_JSON3_str(a) + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} +""" +function Array_to_JSON3_str(data::AbstractArray) + d = Dict("Array"=> data, "size"=>size(data)) + json3_str = JSON3.write(d) + return json3_str +end + +# ---------------------------------------------- 100 --------------------------------------------- # + +""" JSON3_str_to_Array(json3_str::String) + + decode JSON3 String to Array + + # Example + + json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} + a = JSON3_str_to_Array(json3_str) + + a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] +""" +function JSON3_str_to_Array(json3_str::String) + d = JSON3.read(json3_str) + array = reshape(Array(d.Array), (d.size[1], d.size[2])) + return array +end + +# ---------------------------------------------- 100 --------------------------------------------- # + +""" Convert JSON3.read object to OrderedDict + + # Example + dict = dictionary(["a"=>4, "b"=>6]) + OrDict = OrderedDict(dict) + jsonString = JSON3.write(OrDict) # use jsonString to exchange. One can save it to file or send it thru pub/sub + jsonObject = JSON3.read(jsonString) + OrDict2 = JSON3read_to_OrDict(jsonObject) # example here + Adict2 = dictionary(OrDict2) + + Andyferris's github https://github.com/andyferris/Dictionaries.jl +""" +function JSON3read_to_OrDict(x) + dict = OrderedDict() + for (k, v) in x + k = string(k) + dict[k] = v + end + return dict +end + +#------------------------------------------------------------------------------------------------100 + +""" + print time of cpu executtion at the line inwhich this macro is used +""" +macro timeline(expr) + quote + print("line ", $(__source__.line), ": ") + @time $(esc(expr)) + end +end + + +batchindex(batch_counter::Number, batch_size::Number; offset=0) = + (offset + (batch_counter-1) * batch_size + 1) : offset + (batch_counter * batch_size) + +function flip_true_false(x::Bool) + if x == true + x = false + elseif x == false + x = true + else + error("undefined condition line $(@__LINE__)") + end + + return x +end + +function flip_true_false(x::Int) + if x == 1 + x = 0 + elseif x == 0 + x = 1 + else + throw("not define input of type $(typeof(x)) yet") + end + + return x +end + +""" + Return drawed index + # Example + drawed_index = randomWithProb([0.5, 0.2, 0.3]) + +""" +randomWithProb(probability::AbstractVector) = rand(Distributions.Categorical(probability)) # return drawed index + +""" + Draw from choices according to its probability. + Probability range is 0.0 to 1.0 and all probability must summed up to 1 + (may get probability from NNlib's softmax function) + + # Example + + draw = randomChoiceWithProb([true, false, nothing], [0.5, 0.2, 0.3]) +""" +function randomChoiceWithProb(choices::Array, probability::Array) + if length(choices) != length(probability) + error("random is not possible, choices array length != probability array length") + elseif sum(probability) != 1.0 + error("probability does not sum to 1.0") + end + + return choices[randomWithProb(probability)] +end + +function randomChoiceOnTarget(target::Number, targetMatch::Number, choices::AbstractVector, + probability::AbstractVector) + if length(choices) != length(probability) + throw("random is not possible, choices array length != probability array length") + end + return target == targetMatch ? randomChoiceWithProb(choices, probability) : target + # dist = Distributions.Categorical(probability) + # draw_result = choices[rand(dist)] +end + +function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector, + probability::AbstractVector) + return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,)) +end + +function linearly_weighted_avg(a::Array) + total = 0.0 + for (i, v) in enumerate(a) + total = total + (i * v) + end + + return total / sum(a) +end + + +""" Convert String that is holded inside a variable to Symbol +# Example +x = "hello" # x is a variable holding String "hello" \n +y = variable_to_symbol(x) # y holds :hello +""" +function variable_str_to_symbol(variable) + semi = :($variable) + symbol = Symbol(semi) + + return symbol +end + + +""" get useable type of specified fieldname inside a composite struct + + # Example + julia> @Base.kwdef mutable struct some_struct + a::Union{Bool, Nothing} = nothing + b::Union{Float64, Nothing} = nothing + c::Union{Int64, AbstractFloat} = 3.5 + d::Union{String, Nothing} = nothing + end + + julia> a = some_struct() + julia> fieldname_useable_type(some_struct, :c) =result=> [Int64, Float64] +""" +function fieldname_useable_type(somestruct, fieldname::Symbol; + test_types=[2.0, 2, true, :h, "str", 'c', missing, nothing])::Vector{DataType} + new_instance = somestruct() + useable_type = [] + + for i in test_types + try + new_instance.:($fieldname) = i + type = typeof(new_instance.:($fieldname)) + if type ∉ useable_type + push!(useable_type, type) + end + catch + + end + end + + return useable_type +end + + +function randomNoRepeat(drawOptions::Array, draw_number::Integer; + exclude_list::Union{AbstractArray,Nothing}=nothing) + draw_option = copy(drawOptions) + draw_option = isnothing(exclude_list) ? draw_option : + filter!(x -> x ∉ exclude_list, draw_option) + shuffle!(draw_option) + drawed_items = [] + while length(drawed_items) < draw_number + push!(drawed_items, pop!(draw_option)) + end + return drawed_items +end + +""" using cron to schedule backup job by + 1. sudo nano /etc/crontab <<< this is a system-wide cron file + 2. to execute julia file @ 2.00am everyday add the following line at the buttom of the file + 0 2 * * * root julia-1.7 /home/syncthing_backup_script.jl + + Requirements using Dates +""" +function folderBackup(sourceFolderAbsolutePath::String, # absolute path to folder to be backuped + backupFolderAbsolutePath::String; # absolute path to folder used to store backup file + totalBackupFiles::Integer=7, # total backup file, the oldest will be deleted + containerName::Union{Array{String}, Nothing}=nothing) # container using source_folder + + sep = (Sys.iswindows() ? "\\" : '/') + + if sourceFolderAbsolutePath[end] == sep + sourceFolderAbsolutePath = sourceFolderAbsolutePath[1:end-1] + end + if backupFolderAbsolutePath[end] != sl + backupFolderAbsolutePath = backupFolderAbsolutePath * sep + end + + if isdir(backupFolderAbsolutePath) + else + mkpath(backupFolderAbsolutePath) + end + + # stop running docker container service + if containerName !== nothing + println("stop running services") + for i in containerName + try run(`docker stop $i`) catch; end + sleep(10) # wait for services to stop + end + end + + # do backup + println("doing backup now") + timestamp = string(Dates.now()) + name = split(sourceFolderAbsolutePath, sep)[end] * "--" + filename = name * timestamp * ".zip" # resulting compressed filename + run(`chmod -R a+rwx $sourceFolderAbsolutePath`) + # zip -r [destination+filename] [source folder to be zipped] + run(`zip -r $(backupFolderAbsolutePath * filename) $sourceFolderAbsolutePath`) + + # check if total backup file is more than user specified, if yes, delete the oldest backup + backupFiles = readdir(backupFolderAbsolutePath) + + while length(backupFiles) > totalBackupFiles + run(`rm $(backupFolderAbsolutePath * backupFiles[1])`) + backupFiles = readdir(backupFolderAbsolutePath) + end + + # start docker services + if containerName !== nothing + println("start services") + for i in containerName + try run(`docker start $i`) catch; end + sleep(10) # wait for services to stop + end + end +end + +function lowerclip!(data::AbstractVector, lowerbound::Number) + replace!(x -> x < lowerbound ? lowerbound : x, data) +end + +function upperclip!(data::AbstractVector, upperbound::Number) + replace!(x -> x > upperbound ? upperbound : x, data) +end + +function normalise(x::AbstractArray, mu, std) + ϵ = oftype(x[1], 1e-5) + μ = mu + # σ = std(x, dims=dims, mean=μ, corrected=false) # use this when Zygote#478 gets merged + σ = std + return (x .- μ) ./ (σ .+ ϵ) +end + +function minMaxScaler(x::AbstractVector) + min = findmin(x)[1] + max = findmax(x)[1] + + scaler(a::Number, min::Number, max::Number) = (a-min) / (max-min) + return scaler.(x, min, max) +end + +""" a = [-1e200, -1e-200, 1e200, 1e-200] \n + result = vtclamp.(a, 1e-6, 1e6, -1e6, -1e-6) +""" +function customclamp(x::Number, poslo::Number, poshi::Number, + neglo::Number, neghi::Number) + signx = sign(x) + if signx == -1 + if neghi < x < 0 + return neghi + elseif x < neglo + return neglo + else + return x + end + elseif signx == +1 + if poshi < x + return poshi + elseif 0 < x < poslo + return poslo + else + return x + end + end +end + +function unitVec(x::AbstractVector) + y = √(sum(x.^2)) + return x./y +end + +function replaceAt!(x::AbstractVector, ind::AbstractVector, value::Number) + for i in ind + x[i] = value + end +end + +function signbitVec(x::AbstractVector) + sign = signbit.(x) * 1 + signVec = replace(s -> s == 0 ? -1 : s, sign) + return signVec +end + +function deleteall!(x::AbstractVector) + for i in 1:length(x) + deleteat!(x, 1) + end +end + +""" Select specific range of vectors in a dict, return a new dict +# Example +dict = Dict(:a => [1:5...], + :b => [6:10...]) + +call -> selectRange(dict, 1:3) +return -> Dict{Any, Any} with 2 entries: + :a => [1, 2, 3] + :b => [6, 7, 8] +""" +function selectRange(d::Dict{Symbol, <:AbstractVector}, range) + newDict = Dict{Symbol, AbstractVector}() + for (k, v) in d + newDict[k] = v[range] + end + + return newDict +end + +""" Assign value to a given Dict by array of keys + + # Example + d = Dict( + :a1=> Dict(:c=> 5), + :a2=> Dict( + :k=> 10, + :b=> Dict( + :s=> "target", + ) + ) + ) + index = [:a2, :b, :s] \n + assignDict!(d, [:a2, :b, :s], "wow") + + return 1 if no target key in a given dict. +""" +function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign) + wd = nothing + for i in accessArray + println(i) + if i != accessArray[end] + if wd === nothing && haskey(dict, i) + wd = Ref(dict[i]) + elseif wd.x !== nothing && haskey(wd.x, i) + wd = Ref(wd.x[i]) + else + return 1 # error, no target key in a given dict. + end + else + wd.x[i] = valueToAssign + return 0 + end + end +end + +""" convert hour(0-23), minute(0-59) into julia time object +# Example + time +""" +function iTime(h::Integer, m::Integer) + if h == 0 + h = 12 + ampm = "am" + elseif 1 <= h <= 11 + ampm = "am" + elseif h == 12 + ampm = "pm" + elseif 13 <= h <= 23 + h = h - 12 + ampm = "pm" + else + error("hour out of range") + end + + m = m < 10 ? "0$m" : m + t = "$h:$m$ampm" + + return Time(t, "HH:MMp") +end + +""" replace a number according to the limit + if value is lower than lowerbound return lowerbound replacement value + if value is more than upperbound return upperbound replacement value + + # Example + limitvalue(4, (-5 => 0), (5 => 5)) +""" +function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair) + lwLimit, lwReplace = lowerbound + upLimit, upReplace = upperbound + + if v < lwLimit + v = lwReplace + elseif v > upLimit + v = upReplace + else + end + return v +end + + +""" Assign matrix b to matrix a according to matrix b's CartesianIndex. + + Arguments:\n + a : target matrix. + b : source matrix. + + Return:\n + Resulting matrix a. + + Example:\n + ```jldoctest + julia> not done yet + ``` +""" +function cartesianAssign!(a, b) + for (i, v) in enumerate(b) + a[CartesianIndices(b)[i].I...] = v + end + return nothing +end + + +function sumAlongDim3(a::Array) + totalDim = length(size(a)) + + if totalDim == 3 + d1, d2, d3 = size(a) + r = zeros(1, 1, d3) + for i in 1:d3 + view(r, 1, 1, i) .= sum(a[:, :, i]) + end + elseif totalDim == 4 + d1, d2, d3, d4 = size(a) + r = zeros(1, 1, d3, d4) + for j in 1:d4 + for i in 1:d3 + view(r, 1, 1, i, j) .= sum(a[:, :, i, j]) + end + end + else + error("this condition is not define yet") + end + + return r +end + + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all slice of 3D another matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> input = rand(32, 32, 128) # batch at 3rd dim + julia> weight = rand(32, 32, 1024) + julia> r = matMul3Dto3DmanyTo1batch(input, weight); + julia> size(r) + (32, 32, 1024, 128) +""" +function matMul3Dto3DmanyTo1batch(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + asize = [size(a)...] + bsize = [size(b)...] + if resultStorage === nothing + resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) + end + + c = [slice .* b for slice in eachslice(a, dims=3)] + resultStorage .= cat(c..., dims=4) + + return resultStorage +end + + +""" GPU kernel +""" +function matMul3Dto3DmanyTo1batch_gpu!(a, b, resultStorage, linearToCartesian) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + if i <= size(a, 3) # guard against unused threads to accessing memory out of bound + cartesianIndex = linearToCartesian(i, size(b)) # example for how to send "inner" function to gpu + # @cuprintln("gpu thread $i $cartesianIndex[2]") + + @. @views resultStorage[:, :, :, i] = a[ :, :, i] * b + # view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b # alternative code + # @view(resultStorage[:, :, :, i]) .= @view(a[ :, :, i]) .* b # alternative code + end + return nothing +end + + +""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all batch of another 4D matrix ,b, and + concatenate at the 4th dimension. + + Example + julia> + julia> a = rand(2,2,3) # 3-batches + julia> b = rand(2,2,4,3) # 3-batches + julia> r = GeneralUtils.matMul_3Dto4D_batchwise(a, b); + julia> size(r) + (2, 2, 4, 3) +""" +function matMul_3Dto4D_batchwise(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) + if size(a, 3) != size(b, 4) + error("batch number of a and b must be equal") + end + if resultStorage === nothing + resultStorage = zeros(size(b, 1), size(b, 2), size(b, 3), size(a, 3)) + end + for i in 1:size(a, 3) + view(resultStorage, :, :, :, i) .= a[:, :, i] .* b[:, :, :, i] + end + return resultStorage +end + +""" GPU-compatible linear index to cartesian index conversion +""" +function linearToCartesian(i::Int, arraySize::NTuple{4,Int}) + # Check that the linear coordinate is valid + # prod(arraySize) is the same as *(arraySize...). they multipy all elements in an array. + # but this code use prod() because splat breaks GPU performance + if i < 1 || i > prod(arraySize) + error("Invalid linear coordinate") + end + # Extract the dimensions of the matrix + n1, n2, n3, n4 = arraySize + # Compute the cartesian coordinate using rem and div functions + i1 = ((i-1) % (n1)) + 1 # +1 convert 0-based to 1-based index + i2 = ((i-1) ÷ (n1)) % n2 + 1 + i3 = ((i-1) ÷ (n1*n2)) % n3 + 1 + i4 = (i-1) ÷ (n1*n2*n3) + 1 + # Return the cartesian coordinate as a tuple + return (i1, i2, i3, i4) +end + +""" return a vector with true at max value and false for other value. + if vector is all-zeros, return all-false vector. +""" +function vectorMax(x) + if sum(isNotEqual.(x, 0)) == 0 # guard against all-zeros array + # instead of returning all-zeros original vector, + # return all-false vector to prevent type instability + return isNotEqual.(x, 0) + else + return isequal.(x, maximum(x)) + end +end + +function multiply_last(matrix, x, n) + # X is the scalar to multiply + # matrix is the column-major 2D matrix + # n is the number of elements to be multiplied, starting from the last one + # returns a new matrix with the same shape as the original one + + # get the number of rows and columns of the matrix + rows, cols = size(matrix) + + # create a copy of the matrix to avoid mutating the original one + result = copy(matrix) + + # loop over the last n elements in column-major order + for i in (rows * cols - n + 1):(rows * cols) + # get the row and column indices of the current element + row = (i - 1) % rows + 1 + col = (i - 1) ÷ rows + 1 + + # multiply the element by X and store it in the result matrix + result[row, col] *= x + end + + # return the result matrix + return result +end + +function multiplyRandomElements(A, x, n, rng=MersenneTwister(1234)) + # rng is a random number generator object, see https://docs.julialang.org/en/v1/stdlib/Random/ + # x is a scalar value to multiply by + # A is a column-major 2D matrix or a vector + # n is the number of elements to be multiplied + # returns a new array with n randomly chosen distinct elements multiplied by x + B = copy(A) # make a copy of A to avoid mutating it + d = ndims(A) # get the number of dimensions of A + if d == 1 # if A is a vector + m = length(A) # get the length of A + indices = collect(1:m) # create an array of indices from 1 to m + shuffle!(rng, indices) # shuffle the indices in-place using the RNG + for i in 1:n # loop n times + j = indices[i] # get the i-th shuffled index + B[j] *= x # multiply the element at j by x + end + elseif d == 2 # if A is a matrix + m = size(A, 1) # number of rows in A + p = size(A, 2) # number of columns in A + indices = collect(1:m*p) # create an array of linear indices from 1 to m*p + shuffle!(rng, indices) # shuffle the indices in-place using the RNG + for i in 1:n # loop n times + j = indices[i] # get the i-th shuffled index + B[j] *= x # multiply the element at j by x + end + else # if A is neither a vector nor a matrix + error("A must be a vector or a matrix") + end + return B # return the new array +end + +""" Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and + replace matrix A's elements of the same position with value, a. + + Example + julia> mask = rand([-1,0,1],4,4,1) + julia> A = rand(4,4,1) + julia> C = replaceElements(mask, A, -1, 5.0, 3) +""" +function replaceElements(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number, + n::Int=0; rng::AbstractRNG=MersenneTwister(1234)) + """ Prompt + Write a julia function to operate on column-major 3D matrix. The function randomly + choose elements in matrix mask that has value markValue and replace elements in matrix A at + the same position with value a. The choosing randomness is controlled by rng function. + I also want to specify how many elements to be replaced. + """ + + total_x_tobeReplced = sum(isequal.(mask, markValue)) + if n == 0 || n > total_x_tobeReplced + n = total_x_tobeReplced + end + + # check if mask and A have the same size + if size(mask) != size(A) + error("mask and A must have the same size") + end + C = copy(A) + # get the indices of elements in mask that equal markValue + indices = findall(x -> x == markValue, mask) + # shuffle the indices using the rng function + shuffle!(rng, indices) + # select the first n indices + selected = indices[1:n] + # replace the elements in A at the selected positions with a + for i in selected + C[i] = a + end + return C +end + +""" Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and + replace matrix A's elements of the same position with value, a. if n == 0, all marked value is replaced + + Example + julia> mask = rand([-1,0,1],4,4,1) + julia> A = rand(4,4,1) + julia> C = replaceElements(mask, A, -1, 5.0, 3) +""" +function replaceElements!(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number, + n::Int=0; rng::AbstractRNG=MersenneTwister(1234)) + total_x_tobeReplced = sum(isequal.(mask, markValue)) + remaining = 0 + if n == 0 || n > total_x_tobeReplced + remaining = n - total_x_tobeReplced + n = total_x_tobeReplced + end + + # check if mask and A have the same size + if size(mask) != size(A) + error("mask and A must have the same size, mask $(size(mask)) A $(size(A))") + end + # get the indices of elements in mask that equal markValue + indices = findall(x -> x == markValue, mask) + # shuffle the indices using the rng function + shuffle!(rng, indices) + # select the first n indices + selected = indices[1:n] + # replace the elements in A at the selected positions with a + for i in selected + A[i] = a + end + + return remaining +end + +""" Replace n elements that has value x with user specified value a. +""" +function replaceElements(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234)) + total_x_tobeReplced = sum(isequal.(A, x)) + if n == 0 || n > total_x_tobeReplced + n = total_x_tobeReplced + end + + B = copy(A) + # A is a column-major 3D matrix + # x is the value to be replaced + # a is the new value + # rng is a random number generator function + # n is the number of elements to be replaced + + # find the indices of elements in A that equal x + indices = findall(==(x), B) + + # shuffle the indices using the rng function + shuffle!(rng, indices) + + # select the first n indices + selected = indices[1:n] + + # replace the elements at the selected indices with a + for i in selected + B[i] = a + end + + # return the modified matrix A + return B +end + + +function replaceElements!(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234)) + total_x_tobeReplced = sum(isequal.(A, x)) + if n == 0 || n > total_x_tobeReplced + n = total_x_tobeReplced + end + + # A is a column-major 3D matrix + # x is the value to be replaced + # a is the new value + # rng is a random number generator function + # n is the number of elements to be replaced + + # find the indices of elements in A that equal x + indices = findall(==(x), A) + + # shuffle the indices using the rng function + shuffle!(rng, indices) + + # select the first n indices + selected = indices[1:n] + + # replace the elements at the selected indices with a + for i in selected + A[i] = a + end +end + + + + + +""" Get characters between specified characters. + +# Arguments + - `text::T` + a text being searched + - `startChar::Char` + start character + - `endChar::Char` + end character +# Keyword Arguments + - `endCharLocation::String` + end character position after startChar. Can be "next" or "end". "next" means the closed + endChar just after startChar. "end" means the furthest endChar. + - `includeChar::Bool` + whether to include the startChar and endChar. Default is true +# Return + the characters between specified characters. + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> text = "{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}\n\n" +julia> GeneralUtils.getStringBetweenCharacters(text, '{', '}', endCharLocation="end") +"{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}" +``` +# Signature +""" +function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char; + endCharLocation::String="next", includeChar::Bool=true)::String where {T<:AbstractString} + + # get the position of the startChar + startCharPosition = findfirst(startChar, text) + endCharPosition = nothing + + if endCharLocation == "end" + # get the first position of the endChar coming from the end of text + endCharPosition = findlast(endChar, text) + elseif endCharLocation == "next" + # get the first position of the endChar after startCharPosition + endCharPosition = findnext(endChar, text, startCharPosition + 1) + else + error("endCharPositio must be \"end\" or \"next\"") + end + + @show startCharPosition, endCharPosition + + # get characters between startChar and endChar from text + extractedText = text[startCharPosition:endCharPosition] + # convert substring to string + extractedText = string(extractedText) + + extractedText = includeChar == true ? extractedText : extractedText[2:end-1] + + return extractedText +end + + + +""" Read JSON string and return a dictionary with string key. (JSON3 defaults to symbol key) + This function solve the problem of reading JSON with string key. + + Arguments: + jsonString::String + + Return: + a dictionary with string key + + Example: + ```jldoctest + julia> jsonString = {\"wine type\": \"Red\", \"intensity level\": \"medium-bodied\"} + julia> JSON3read_stringKey(jsonString) + Dict{String, Any} with 2 entries: + "intensity level" => "medium-bodied" + "wine type" => "Red" + ``` +""" +function JSON3read_stringKey(jsonString::AbstractString) + jsonobj = JSON3.read(jsonString) + newDict = OrderedDict{String,Any}() + for (k,v) in jsonobj + newDict[string(k)] = v + end + return newDict +end + + +""" Create nested dict path if it does not already exist. The same concept as Julia's mkpath() + +# Arguments + - `dict::Dict` + target dict + - `addkeys::Union{Vector{String}, Vector{Symbol}}` + keys to be added to dict + - `value` + value to be added to dict at final key in keypath + +# Return + - dict with added keypath + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> d = Dict{String, Any}("a" => Dict{String, Any}("b" => 10)) +julia> GeneralUtils.mkDictPath!(d, ["a", "v", "x", "y", "z"], 42) +Dict{String, Any} with 1 entry: +"path" => Dict{Any, Any}("to"=>Dict{Any, Any}("nested"=>Dict{Any, Any}("value"=>42))) +``` + +# Signature +""" +function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}}, + addkeys::Union{Vector{String}, Vector{Symbol}}, value) + # new key and existing key must be the same type + if !isempty(dict) + existingKeys = [key for key in keys(dict)] + if typeof(existingKeys[1]) != typeof(addkeys[1]) + error("Type of keys being added is $(typeof(addkeys[1])) but type of existing keys is $(typeof(existingKeys[1]))") + end + end + + for key in addkeys[1:end-1] + if !haskey(dict, key) + key_type = eltype(keys(dict)) + dict[key] = Dict{key_type, Any}() + end + dict = dict[key] + end + + return dict[addkeys[end]] = value +end + + +""" Get nested dict value using a vector of keys + +# Arguments + - `dict::Dict` + target dict + - `keys::Vector` + keys vector + +# Return + - dict with added keypath + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> d = Dict{Symbol, Any}(:a => Dict{Symbol, Any}(:b => 10)) +julia> GeneralUtils.getDictPath(d, [:a, :b]) +10 +``` + +# Signature +""" +function getDictPath(dict::Dict, keys::Vector) + current_dict = dict + for key in keys[1:end-1] + if haskey(current_dict, key) + current_dict = current_dict[key] + else + throw(ArgumentError("Key $key not found in dictionary")) + end + end + + last_key = keys[end] + if haskey(current_dict, last_key) + return current_dict[last_key] + else + throw(ArgumentError("Key $last_key not found in dictionary")) + end +end + + +""" Convert a dataframe into CSV. + +# Arguments + - `df::DataFrame` + A connection object to Postgres database + +# Return + - `result::String` + +# Example +```jldoctest +julia> using DataFrames, GeneralUtils +julia> df = DataFrame(A=1:3, B=5:7, fixed=1) +julia> result = GeneralUtils.dataframeToCSV(df) +``` + +# Signature +""" +function dataframeToCSV(df::DataFrame) + # Create an IOBuffer to capture the output + io = IOBuffer() + CSV.write(io, df) + dfStr = String(take!(io)) + return dfStr +end + + + + + + + + + +end # module \ No newline at end of file diff --git a/src/llmUtil.jl b/src/llmUtil.jl new file mode 100644 index 0000000..ccc570b --- /dev/null +++ b/src/llmUtil.jl @@ -0,0 +1,261 @@ +module llmUtil + +export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection + +using UUIDs, JSON3, Dates +using GeneralUtils + +# ---------------------------------------------- 100 --------------------------------------------- # + + +""" Convert a single chat dictionary into LLM model instruct format. + +# Llama 3 instruct format example + <|begin_of_text|> + <|start_header_id|>system<|end_header_id|> + You are a helpful assistant. + <|eot_id|> + <|start_header_id|>user<|end_header_id|> + Get me an icecream. + <|eot_id|> + <|start_header_id|>assistant<|end_header_id|> + Go buy it yourself at 7-11. + <|eot_id|> + +# Arguments + - `name::T` + message owner name e.f. "system", "user" or "assistant" + - `text::T` + +# Return + - `formattedtext::String` + text formatted to model format + +# Example +```jldoctest +julia> using Revise +julia> using YiemAgent +julia> d = Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",) +julia> formattedtext = YiemAgent.formatLLMtext_llama3instruct(d[:name], d[:text]) +"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n" +``` + +Signature +""" +function formatLLMtext_llama3instruct(name::T, text::T) where {T<:AbstractString} + formattedtext = + if name == "system" + """ + <|start_header_id|>$name<|end_header_id|> + $text + <|eot_id|> + """ + else + """ + <|start_header_id|>$name<|end_header_id|> + $text + <|eot_id|> + """ + end + + return formattedtext +end +# function formatLLMtext_llama3instruct(name::T, text::T) where {T<:AbstractString} +# formattedtext = +# if name == "system" +# """ +# <|begin_of_text|> +# <|start_header_id|>$name<|end_header_id|> +# $text +# <|eot_id|> +# """ +# else +# """ +# <|start_header_id|>$name<|end_header_id|> +# $text +# <|eot_id|> +# """ +# end + +# return formattedtext +# end + + + +""" Convert a chat messages in vector of dictionary into LLM model instruct format. + +# Arguments + - `messages::Vector{Dict{Symbol, T}}` + message owner name e.f. "system", "user" or "assistant" + - `formatname::T` + format name to be used + +# Return + - `formattedtext::String` + text formatted to model format + +# Example +```jldoctest +julia> using Revise +julia> using YiemAgent +julia> chatmessage = [ + Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",), + Dict(:name=> "user",:text=> "list me all planets in our solar system.",), + Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",), + ] +julia> formattedtext = YiemAgent.formatLLMtext(chatmessage, "llama3instruct") +"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n <|start_header_id|>user<|end_header_id|>\n list me all planets in our solar system.\n <|eot_id|>\n <|start_header_id|>assistant<|end_header_id|>\n I'm sorry. I don't know. You tell me.\n <|eot_id|>\n" +``` + +# Signature +""" +function formatLLMtext(messages::Vector{Dict{Symbol, T}}; formatname::String="llama3instruct" + )::String where {T<:Any} + f = if formatname == "llama3instruct" + formatLLMtext_llama3instruct + elseif formatname == "mistral" + # not define yet + elseif formatname == "phi3instruct" + formatLLMtext_phi3instruct + else + error("$formatname template not define yet") + end + + str = "" + for t in messages + str *= f(t[:name], t[:text]) + end + + # add <|assistant|> so that the model don't generate it and I don't need to clean it up later + if formatname == "phi3instruct" + str *= "<|assistant|>\n" + end + + return str +end + + +""" Attemp to correct LLM response's incorrect JSON response. + +# Arguments + - `a::T1` + one of Yiem's agent + - `input::T2` + text to be send to virtual wine customer + +# Return + - `correctjson::String` + corrected json string + +# Example +```jldoctest +julia> +``` + +# Signature +""" +function jsoncorrection(config::T1, input::T2, correctJsonExample::T3; + maxattempt::Integer=3 + ) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString} + + incorrectjson = deepcopy(input) + correctjson = nothing + + for attempt in 1:maxattempt + try + d = copy(JSON3.read(incorrectjson)) + correctjson = incorrectjson + return correctjson + catch e + @warn "Attempting to correct JSON string. Attempt $attempt" + e = """$e""" + if occursin("EOF", e) + e = split(e, "EOF")[1] * "EOF" + end + incorrectjson = deepcopy(input) + _prompt = + """ + Your goal are: + 1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function. + 2) Provide Corrected JSON string only. Do not provide any other info. + + $correctJsonExample + + Let's begin! + Given JSON string: $incorrectjson + The given JSON string failed to load previously because: $e + Corrected JSON string: + """ + + # apply LLM specific instruct format + externalService = config[:externalservice][:text2textinstruct] + llminfo = externalService[:llminfo] + prompt = + if llminfo[:name] == "llama3instruct" + formatLLMtext_llama3instruct("system", _prompt) + else + error("llm model name is not defied yet $(@__LINE__)") + end + + # send formatted input to user using GeneralUtils.sendReceiveMqttMsg + msgMeta = GeneralUtils.generate_msgMeta( + externalService[:mqtttopic], + senderName= "jsoncorrection", + senderId= string(uuid4()), + receiverName= "text2textinstruct", + mqttBroker= config[:mqttServerInfo][:broker], + mqttBrokerPort= config[:mqttServerInfo][:port], + ) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :text=> prompt, + :kwargs=> Dict( + :max_tokens=> 512, + :stop=> ["<|eot_id|>"], + ) + ) + ) + result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120) + incorrectjson = result[:response][:text] + end + end +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module llmUtil \ No newline at end of file diff --git a/src/util.jl b/src/util.jl new file mode 100644 index 0000000..5c77de2 --- /dev/null +++ b/src/util.jl @@ -0,0 +1,387 @@ +module util + +export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, replaceDictKeys, + findMatchingDictKey, textToDict, randstring, randstrings + +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient + +# ---------------------------------------------- 100 --------------------------------------------- # + +""" Compute time different between start time and stop time in a given unit. + Unit can be "milliseconds", "seconds", "minutes", "hours". + +# Arguments + - `starttime::DateTime` + start time + - `stoptime::DateTime` + stop time + - `unit::String` + unit of time difference + +# Return + - time difference in given unit + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils, Dates +julia> a = Dates.now() +julia> b = a + Dates.Day(5) # add 5 days +julia> GeneralUtils.timedifference(a, b, "hours") +120 +``` + +# Signature +""" +function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer + diff = stoptime - starttime + unit = lowercase(unit) + + if unit == "milliseconds" + return diff.value + elseif unit == "seconds" + return diff.value ÷ 1000 + elseif unit == "minutes" + return diff.value ÷ (1000 * 60) + elseif unit == "hours" + return diff.value ÷ (1000 * 60 * 60) + else + error("Invalid unit specified. Please choose from: milliseconds, seconds, minutes, hours") + end +end + + +""" Capture then show error and stacktrace + +# Arguments + - `f::Function` + a function that might throws an error + - `args` function f arguments + +# Return + - `outcome::NamedTuple` + (success, result, errormsg, st) + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils, PrettyPrinting +julia> testf(a, b) = a + b +julia> success, result, errormsg, st = GeneralUtils.showstracktrace(testf, 5, "6") +julia> pprint(st) +16-element Vector{Base.StackTraces.StackFrame}: +testf(a::Int64, b::String) at REPL[12]:1 +showstracktrace(::Function, ::Int64, ::Vararg{Any}) at util.jl:95 +... +``` + +# Signature +""" +function showstracktrace(f::Function, args...)::NamedTuple + global st = nothing # stacktrace + global errorMsg = nothing + global success = false + global fResult = nothing + + try + success, fResult + fResult = f(args...) + success = true + catch e + io = IOBuffer() + showerror(io, e) + errorMsg = String(take!(io)) + + st = sprint((io, v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) + @warn "Error occurred: $errorMsg\n$st" + end + + return (success=success, result=fResult, errormsg=errorMsg, st=st) +end + +""" Find all match key of a dictionary for a given key. + +# Arguments + - `d<:AbstractDict` + The dictionary to search for keys. + - `text<:Symbol` + The text to match against the keys. + +# Returns + - `result::Vector{Symbol}` + A vector of matched key + + +# Examples +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> d = Dict(:key_1 => "apple", :key_12 => "banana", :key_3 => "cherry") +julia> GeneralUtils.findMatchingDictKey(d, "key_1") +2-element Vector{Symbol}: + :key_1 + :key_12 +``` + +# Signature +""" +function findMatchingDictKey(d::T, text::Union{String, Symbol} + )::Vector{Symbol} where {T<:AbstractDict} + + _matching_keys = filter(k -> occursin(string(text), string(k)), keys(d)) + matching_keys = collect(_matching_keys) # convert from Set into Array + + return matching_keys +end + + +""" +Find the key in a dictionary `d` with the highest index value that matches a given `text`. + +# Arguments + - `d<:AbstractDict` + The dictionary to search for keys. + - `text<:Union{String, Symbol}` + The text to match against the keys. + +# Returns + - `NamedTuple{(:result, :maxindice), Tuple{Union{Symbol, Nothing}, Union{Integer, Nothing}}}` + The key in `d` with the highest index value that matches `text`, or `nothing` if no matches are found. + +# Examples +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> d = Dict(:key_1 => "apple", :key_2 => "banana", :key_3 => "cherry") +julia> GeneralUtils.findHighestIndexKey(d, "key") +(:key_3, 3) +``` + +# Signature +""" +function findHighestIndexKey(d::T, text::Union{String, Symbol} + )::NamedTuple{(:result, :maxindice), Tuple{Union{Symbol, Nothing}, Union{Integer, Nothing}}} where {T<:AbstractDict} + + matching_keys = findMatchingDictKey(d, text) + + if isempty(matching_keys) + return (result=nothing, maxindice=nothing) + elseif length(matching_keys) == 1 && matching_keys[1] == Symbol(text) + return (result=Symbol(text), maxindice=nothing) + else + indices = parse.(Int, replace.(string.(matching_keys), r"[^\d]" => "")) + maxIndexKey = matching_keys[argmax(indices)] + return (result=maxIndexKey, maxindice=maximum(indices)) + end +end + + + +""" Get uuid4 with snake case + +# Return + - `uuid4::String` + uuid4 with snake case + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> GeneralUtils.uuid4snakecase() +"0f6e4f_568c_4df4_8c79_1d7a58072f4a" +``` + +# Signature +""" +function uuid4snakecase()::String + _id = string(uuid4()) + id = replace(_id, "-" => "_") + return id +end + + +""" Replace a dictionary key with the new key + +# Arguments + - `d::Dict` + The input dictionary that you want to modify + - `replacementMap::Dict` + A dictionary that maps old keys to new keys + +# Return + - `newDict::Dict` + new dictionary with the replaced keys + +# Example +```jldoctest +julia> using Revise +julia> using GeneralUtils +julia> d = Dict(:a => 1, :b => 2, :c => 3) +julia> replacement_map = Dict(:a => :x, :b => :y) +julia> new_dict = GeneralUtils.replaceDictKeys(d, replacement_map) +Dict{Any, Any} with 3 entries: + :y => 2 + :c => 3 + :x => 1 +``` + +# Signature +""" +function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict + newDict = Dict() + for (key, value) in d + newKey = get(replacementMap, key, key) # Get the replacement key if it exists, otherwise keep the original key + newDict[newKey] = value + end + return newDict +end + + +""" Convert text into a dictionary with a given keywords. This function use keywords to slice + a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text. + The left most string which has no keyword will be discarded. WARNING, ordering is important + +# Arguments + - `text::String` + A text to be converted. + - `keywords::Vector{String}` + A list of keywords to be used to slice the text. + These keywords also be the resulting dict keys. +# Keyword Arguments + - `rightmarker::String` + A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", + the function will search for "plan:" otherwise the function will search for "plan". + The marker will not be in the resulting dict keys. + - `symbolkey::Bool` + If true, resulting dict's key will be Symbols, otherwise string. + - `lowercasekey::Bool` + set resulting dict's key to be lowercase + +# Return + - `d::OrderedDict` + +# Example +```jldoctest +julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep" +julia> sample_keywords = ["thought", "plan", "action"] +julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true) +julia> println(resultdict) +OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", + :plan => "wake up and going out", + :action => "1. wake up 2. eat 3. sleep") +``` + +# Signature +""" +function textToDict(text::String, keywords::Vector{String}; + rightmarker::Union{String, Nothing}=nothing, symbolkey::Bool=false, lowercasekey::Bool=false + )::OrderedDict + + od1, od2 = + if symbolkey + OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}() + else + OrderedDict{String, Any}(), OrderedDict{String, Any}() + end + + remainingtext = text + + for keyword in reverse(keywords) + mkeyword = rightmarker !== nothing ? keyword * rightmarker : keyword + + # Find the position of the keyword in the text + keywordidx = findlast(mkeyword, remainingtext) + + if keywordidx !== nothing + substr = remainingtext[keywordidx[end]+1:end] + str = string(strip(substr)) # Removes both leading and trailing whitespace. + _key = lowercasekey == true ? lowercase(keyword) : keyword + key = symbolkey == true ? Symbol(_key) : _key + od1[key] = str + remainingtext = remainingtext[1:keywordidx[1]-1] + else + error("""keyword "$keyword" not found in the provided text""") + end + end + + keywords = lowercasekey == true ? lowercase.(keywords) : keywords + + # correct the order + for keyword in keywords + key = symbolkey == true ? Symbol(keyword) : keyword + od2[key] = od1[key] + end + + return od2 +end + + + + + +""" Generate a random string + +# Arguments + - `n::Integer` + A number of string to be generated + +# Return + - `s::String` + +# Example +```jldoctest +julia> result = randstring(5) +"fysmp" +``` + +# Signature +""" +randstring(n::Integer)::String = String(rand('a':'z', n)) + + +""" Generate a random string in group + +# Arguments + - `totalgroup::Integer` + A number of group of random string to be generated + - `stringlength::Integer` + A number of string to be generated + +# Return + - `s::String` + +# Example +```jldoctest +julia> result = randstrings(3, 5) +"fysmp cmhdk iuytr" +``` + +# Signature +""" +function randstrings(totalgroup::Integer, stringlength::Integer)::String + str = "" + for i in 1:totalgroup + str *= randstring(stringlength) * " " + end + str = strip(str) + return str +end + + + + + + + + + + + + + + + + + + +end # module util \ No newline at end of file diff --git a/test/runtest.jl b/test/runtest.jl new file mode 100644 index 0000000..fb7cb49 --- /dev/null +++ b/test/runtest.jl @@ -0,0 +1,67 @@ +using Revise +using GeneralUtils + + + +# text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep" + +# sample_keywords = ["thought", "plan", "action"] + + +# resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true) + + +# println(resultdict) + + +storedata = Channel(8) +keepalive = Channel(8) + +function onMsgCallback_1(topic, payload) + jobj = JSON3.read(String(payload)) + incomingMqttMsg = copy(jobj) # convert json object into julia dictionary recursively + + topic = split(topic, "/") + if "keepalive" ∈ _topic + put!(keepaliveChannel, incomingMqttMsg) + elseif "testingprompt" ∈ topic + # @show incomingMqttMsg + put!(storedata, incomingMqttMsg) +else + @show "undefined condition" + end +end + +mqttInstance = GeneralUtils.mqttClientInstance( + "mqtt.yiem.cc", + 1883, + ["/testingprompt"], + storedata, + keepalive, + onMsgCallback_1 +) + + +_ = GeneralUtils.checkMqttConnection!(mqttInstance) + + +println("555") + + + + + + + + + + + + + + + + + + +