commit 9fb5e046ce0cb242eb8911365e4d8d75d95b2c22 Author: narawat Date: Mon Jan 26 07:14:29 2026 +0700 1st push diff --git a/app/.vscode/launch.json b/app/.vscode/launch.json new file mode 100644 index 0000000..bafa6ab --- /dev/null +++ b/app/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "julia", + "request": "launch", + "name": "Run active Julia file", + "program": "${file}", + "stopOnEntry": false, + "cwd": "${workspaceFolder}", + "env": {}, + "juliaEnv": "${command:activeJuliaEnvironment}" + } + ] +} \ No newline at end of file diff --git a/app/.vscode/settings.json b/app/.vscode/settings.json new file mode 100644 index 0000000..9c03966 --- /dev/null +++ b/app/.vscode/settings.json @@ -0,0 +1,20 @@ +{ + "MicroPython.executeButton": [ + { + "text": "▶", + "tooltip": "Run", + "alignment": "left", + "command": "extension.executeFile", + "priority": 3.5 + } + ], + "MicroPython.syncButton": [ + { + "text": "$(sync)", + "tooltip": "sync", + "alignment": "left", + "command": "extension.execute", + "priority": 4 + } + ] +} \ No newline at end of file diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 0000000..31d51d6 --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,103 @@ +# FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 +FROM julia:1.11 +# FROM debian:latest + +# ---------------------------------------------- 100 --------------------------------------------- # + +# install required APT packages +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y \ + software-properties-common \ + build-essential \ + busybox \ + g++ \ + gcc \ + wget \ + net-tools \ + curl \ + iputils-ping \ + unzip \ + unixodbc \ + unixodbc-dev \ + libicu-dev \ + nano \ + libmosquitto-dev \ + git \ + ffmpeg \ + libsm6 \ + libxext6 \ + tar \ + zip \ + libssl-dev \ + # python3 \ + # python3-pip \ + # python-is-python3 \ + postgresql-client \ + cargo \ + procps + +# # For webapp frontend +# RUN apt-get update && apt-get install -y nginx +# COPY nginx.conf /etc/nginx/nginx.conf +# # Copy your static website files to the Nginx HTML directory +# COPY . /usr/share/nginx/html + +# # install nodejs https://deb.nodesource.com/ +# RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && \ +# apt-get update && apt-get install -y nodejs + +# set up the app +RUN mkdir /appfolder +RUN mkdir /appfolder/mountvolume +RUN mkdir /appfolder/app +RUN mkdir /appfolder/app/temp +COPY . /appfolder/app/temp +RUN mv /appfolder/app/temp/env_preparation.jl /appfolder/app + + +# install Conda as primary python environment with specified python version +WORKDIR /appfolder/app/temp +# RUN PATH="${HOME}/conda/bin:${PATH}" \ +# # && wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ +# && wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \ +# && bash Miniforge3.sh -b -p "${HOME}/conda" \ +# && rm -f Miniforge3.sh \ +# && echo "Running $(conda --version)" \ +# && conda init bash \ +# && . /root/.bashrc \ +# && conda update conda -y \ +# && conda install python=3.10 -y + +# # install pip into conda's base env +# RUN PATH="${HOME}/conda/bin:${PATH}" \ +# && conda install pip \ +# && pip install --trusted-host pypi.python.org -r required_python_packages.txt \ +# # && CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS -DLLAMA_AVX2=OFF -DLLAMA_F16C=OFF -DLLAMA_FMA=OFF" FORCE_CMAKE=1 pip install llama-cpp-python --no-cache-dir +# && CMAKE_ARGS="-DLLAMA_CUBLAS=on -DLLAMA_AVX2=OFF -DLLAMA_F16C=OFF -DLLAMA_FMA=OFF" FORCE_CMAKE=1 pip install llama-cpp-python==0.2.77 --no-cache-dir +# # https://github.com/abetlen/llama-cpp-python/issues/412 old CPU cause CUBLAS compile problem + +# # to be able to use CMD python -m at the last line +# ENV PATH=/root/conda/bin:$PATH + +# # install powershell +# WORKDIR /appfolder/app/temp +# RUN wget https://github.com/PowerShell/PowerShell/releases/download/v7.4.5/powershell_7.4.5-1.deb_amd64.deb +# RUN dpkg -i powershell_7.4.5-1.deb_amd64.deb +# RUN apt-get install -f + +# using juliaup +# RUN curl -fsSL https://install.julialang.org | sh -s -- -y +# SHELL ["/bin/bash", "--login" , "-c"] + +# install julia package for my app +WORKDIR /appfolder/app +RUN julia env_preparation.jl +RUN rm -r /appfolder/app/temp + + +# Make port 80 available to the world outside this container. If I use --publish at docker run command, there is no need to use EXPOSE +# EXPOSE 1883 + +# Run app when the container launches +CMD julia -t auto --project -e 'include("main.jl");' +# CMD ["nginx", "-g", "daemon off;"] +# CMD [ "sleep", "infinity" ] diff --git a/app/Manifest.toml b/app/Manifest.toml new file mode 100644 index 0000000..73acf73 --- /dev/null +++ b/app/Manifest.toml @@ -0,0 +1,989 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.4" +manifest_format = "2.0" +project_hash = "f7df91fbf8ec834c31f5a35e27b7a85f5739d7d3" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgCheck]] +git-tree-sha1 = "f9e9a66c9b7be1ad7372bbd9b062d9230c30c5ce" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.5.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.ArrowTypes]] +deps = ["Sockets", "UUIDs"] +git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101" +uuid = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +version = "2.3.0" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.BufferedStreams]] +git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643" +uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" +version = "1.2.2" + +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.15" + +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "b7231a755812695b8046e8471ddc34c8268cbad5" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "3.0.0" + +[[deps.CodecBase]] +deps = ["TranscodingStreams"] +git-tree-sha1 = "40956acdbef3d8c7cc38cba42b56034af8f8581a" +uuid = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf" +version = "0.3.4" + +[[deps.CodecInflate64]] +deps = ["TranscodingStreams"] +git-tree-sha1 = "d981a6e8656b1e363a2731716f46851a2257deb7" +uuid = "6309b1aa-fc58-479c-8956-599a07234577" +version = "0.1.3" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.8" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.Compiler]] +git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" +version = "0.1.1" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.5.0" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DBInterface]] +git-tree-sha1 = "a444404b3f94deaa43ca2a58e18153a82695282b" +uuid = "a10d1c49-ce27-4219-8d33-6db1a4562965" +version = "2.6.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "d8928e9169ff76c6281f39a659f9bca3a573f24c" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.8.1" + +[[deps.DataStructures]] +deps = ["OrderedCollections"] +git-tree-sha1 = "e357641bb3e0638d353c4b29ea0e40ea644066a6" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.19.3" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.Decimals]] +git-tree-sha1 = "e98abef36d02a0ec385d68cd7dadbce9b28cbd88" +uuid = "abce61dc-4473-55a0-ba07-351d65e31d42" +version = "0.4.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.Distributions]] +deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "fbcc7610f6d8348428f722ecbe0e6cfe22e672c6" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.123" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + DistributionsTestExt = "Test" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.11" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "d60eb76f37d7e5a40cc2e7c36974d864b82dc802" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.17.1" +weakdeps = ["HTTP"] + + [deps.FileIO.extensions] + HTTPExt = "HTTP" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.24" +weakdeps = ["Mmap", "Test"] + + [deps.FilePathsBase.extensions] + FilePathsBaseMmapExt = "Mmap" + FilePathsBaseTestExt = "Test" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.16.0" + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStaticArraysExt = "StaticArrays" + FillArraysStatisticsExt = "Statistics" + + [deps.FillArrays.weakdeps] + PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GeneralUtils]] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"] +git-tree-sha1 = "e28ca4df47d0c46d04716422bef6adb660f33dc3" +repo-rev = "main" +repo-url = "https://git.yiem.cc/ton/GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +version = "0.3.1" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "5e6fe50ae7f23d171f44e311c2960294aaa0beb5" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.19" + +[[deps.HashArrayMappedTries]] +git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" +uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" +version = "0.2.0" + +[[deps.HypergeometricFunctions]] +deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.28" + +[[deps.ICU_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b3d8be712fbf9237935bde0ce9b5a736ae38fc34" +uuid = "a51ab1cf-af8e-5615-a023-bc2c838bba6b" +version = "76.2.0+0" + +[[deps.Infinity]] +deps = ["Dates", "Random", "Requires"] +git-tree-sha1 = "cf8234411cbeb98676c173f930951ea29dca3b23" +uuid = "a303e19e-6eb4-11e9-3b09-cd9505f79100" +version = "0.2.4" + +[[deps.InlineStrings]] +git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.5" +weakdeps = ["ArrowTypes", "Parsers"] + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + +[[deps.InputBuffers]] +git-tree-sha1 = "e5392ea00942566b631e991dd896942189937b2f" +uuid = "0c81fc1b-5583-44fc-8770-48be1e1cca08" +version = "1.1.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.Intervals]] +deps = ["ArrowTypes", "Dates", "Printf", "RecipesBase", "Serialization", "TimeZones"] +git-tree-sha1 = "d6fe00b123e32ddd17231b35d69a6394e696fd5a" +uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5" +version = "1.11.0" + +[[deps.InvertedIndices]] +git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.1" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.6" + +[[deps.IterTools]] +git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.10.0" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.1" + +[[deps.JSON]] +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "b3ad4a0255688dcb895a52fafbaae3023b588a90" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "1.4.0" +weakdeps = ["ArrowTypes"] + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.14.3" +weakdeps = ["ArrowTypes"] + + [deps.JSON3.extensions] + JSON3ArrowExt = ["ArrowTypes"] + +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +git-tree-sha1 = "80580012d4ed5a3e8b18c7cd86cebe4b816d17a6" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.10.9" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.Kerberos_krb5_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "0f2899fdadaab4b8f57db558ba21bdb4fb52f1f0" +uuid = "b39eb1a6-c29a-53d7-8c32-632cd16f18da" +version = "1.21.3+0" + +[[deps.LRUCache]] +git-tree-sha1 = "5519b95a490ff5fe629c4a7aa3b3dfc9160498b3" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.2" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.LayerDicts]] +git-tree-sha1 = "6087ad3521d6278ebe5c27ae55e7bbb15ca312cb" +uuid = "6f188dcb-512c-564b-bc01-e0f76e72f166" +version = "1.0.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibPQ]] +deps = ["CEnum", "DBInterface", "Dates", "Decimals", "DocStringExtensions", "FileWatching", "Infinity", "Intervals", "IterTools", "LayerDicts", "LibPQ_jll", "Libdl", "Memento", "OffsetArrays", "SQLStrings", "Tables", "TimeZones", "UTCDateTimes"] +git-tree-sha1 = "3d227cd13cbf1e9a54d7748dab33e078da6f9168" +uuid = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1" +version = "1.18.0" + +[[deps.LibPQ_jll]] +deps = ["Artifacts", "ICU_jll", "JLLWrappers", "Kerberos_krb5_jll", "Libdl", "OpenSSL_jll", "Zstd_jll"] +git-tree-sha1 = "7757f54f007cc0eb516a5000fb9a6fc19a49da7e" +uuid = "08be9ffa-1c94-5ee5-a977-46a84ec9b350" +version = "16.8.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.12.0" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.29" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.2.0" + +[[deps.LoweredCodeUtils]] +deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] +git-tree-sha1 = "65ae3db6ab0e5b1b5f217043c558d9d1d33cc88d" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +version = "3.5.0" + +[[deps.MIMEs]] +git-tree-sha1 = "c64d943587f7187e751162b3b84445bbbd79f691" +uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +version = "1.1.0" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8" +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.1010+0" + +[[deps.Memento]] +deps = ["Dates", "Distributed", "Requires", "Serialization", "Sockets", "Test", "UUIDs"] +git-tree-sha1 = "bb2e8f4d9f400f6e90d57b34860f6abdc51398e5" +uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" +version = "1.4.1" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.Mocking]] +deps = ["Compat", "ExprTools"] +git-tree-sha1 = "2c140d60d7cb82badf06d8783800d0bcd1a7daa2" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.8.1" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2025.11.4" + +[[deps.NATS]] +deps = ["Base64", "BufferedStreams", "CodecBase", "Dates", "DocStringExtensions", "JSON3", "MbedTLS", "NanoDates", "Random", "ScopedValues", "Sockets", "Sodium", "StructTypes", "URIs"] +git-tree-sha1 = "d9d9a189fb9155a460e6b5e8966bf6a66737abf8" +uuid = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" +version = "0.1.0" + +[[deps.NanoDates]] +deps = ["Dates", "Parsers"] +git-tree-sha1 = "850a0557ae5934f6e67ac0dc5ca13d0328422d1f" +uuid = "46f1a544-deae-4307-8689-c12aa3c955c6" +version = "1.0.3" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "117432e406b5c023f665fa73dc26e79ec3630151" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.17.0" + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + + [deps.OffsetArrays.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.29+0" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.7+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "NetworkOptions", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "1d1aaa7d449b58415f97d2839c318b70ffb525a0" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.6.1" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.4+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.Oxygen]] +deps = ["DataStructures", "Dates", "HTTP", "JSON", "LRUCache", "MIMEs", "Reexport", "RelocatableFolders", "Sockets", "Statistics", "StructTypes"] +git-tree-sha1 = "b0a48def13f76870688eb450096e79fbecf914c2" +uuid = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" +version = "1.10.0" + + [deps.Oxygen.extensions] + BonitoExt = "Bonito" + CairoMakieExt = "CairoMakie" + MustacheExt = "Mustache" + OteraEngineExt = "OteraEngine" + ProtoBufExt = "ProtoBuf" + TimeZonesExt = "TimeZones" + WGLMakieExt = ["WGLMakie", "Bonito"] + + [deps.Oxygen.weakdeps] + Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8" + CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" + Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" + OteraEngine = "b2d7f28f-acd6-4007-8b26-bc27716e5513" + ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429" + TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" + WGLMakie = "276b4fcb-3e11-5398-bf8b-a0c2d153d008" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "e4cff168707d441cd6bf3ff7e4832bdf34278e4a" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.37" +weakdeps = ["StatsBase"] + + [deps.PDMats.extensions] + StatsBaseExt = "StatsBase" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.3" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.3.3" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.1" + +[[deps.PrettyPrinting]] +git-tree-sha1 = "142ee93724a9c5d04d78df7006670a93ed1b244e" +uuid = "54e16d92-306c-5ea0-a30b-337be88ac337" +version = "0.4.2" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "c5a07210bd060d6a8491b0ccdee2fa0235fc00bf" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "3.1.2" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.ProgressMeter]] +deps = ["Distributed", "Printf"] +git-tree-sha1 = "fbb92c6c56b34e1a2c4c36058f68f332bec840e7" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.11.0" + +[[deps.PtrArrays]] +git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.3.0" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.11.2" + + [deps.QuadGK.extensions] + QuadGKEnzymeExt = "Enzyme" + + [deps.QuadGK.weakdeps] + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.RelocatableFolders]] +deps = ["SHA", "Scratch"] +git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864" +uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" +version = "1.0.1" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.Revise]] +deps = ["CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Preferences", "REPL", "UUIDs"] +git-tree-sha1 = "dfd6fab9aa325b382773b8930998ce84c2918e5e" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.13.1" +weakdeps = ["Distributed"] + + [deps.Revise.extensions] + DistributedExt = "Distributed" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.9.0" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.5.1+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SQLStrings]] +git-tree-sha1 = "55de0530689832b1d3d43491ee6b67bd54d3323c" +uuid = "af517c2e-c243-48fa-aab8-efac3db270f5" +version = "0.1.0" + +[[deps.ScopedValues]] +deps = ["HashArrayMappedTries", "Logging"] +git-tree-sha1 = "c3b2323466378a2ba15bea4b2f73b081e022f473" +uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" +version = "1.5.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.3.0" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "ebe7e59b37c400f694f52b58c93d26201387da70" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.9" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.Sodium]] +deps = ["Base64", "libsodium_jll"] +git-tree-sha1 = "907703e0d50846f300650d7225bdcab145b7bca9" +uuid = "4f5b5e99-b0ad-42cd-b47a-334e172ec8bd" +version = "1.1.2" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.2" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.12.0" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "f2685b435df2613e25fc10ad8c26dddb8640f547" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.6.1" + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + + [deps.SpecialFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.8.0" + +[[deps.StatsBase]] +deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "aceda6f4e598d331548e04cc6b2124a6148138e3" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.10" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.5.2" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a3c1536470bf8c5e02096ad4853606d7c8f62721" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.2" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.11.0" + +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "9297459be9e338e546f5c4bedb59b3b5674da7f1" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.6.2" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.8.3+2" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TZJData]] +deps = ["Artifacts"] +git-tree-sha1 = "72df96b3a595b7aab1e101eb07d2a435963a97e2" +uuid = "dc5dba14-91b3-4cab-a142-028a31da12f7" +version = "1.5.0+2025b" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimeZones]] +deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"] +git-tree-sha1 = "d422301b2a1e294e3e4214061e44f338cafe18a2" +uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" +version = "1.22.2" +weakdeps = ["RecipesBase"] + + [deps.TimeZones.extensions] + TimeZonesRecipesBaseExt = "RecipesBase" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIs]] +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.6.1" + +[[deps.UTCDateTimes]] +deps = ["Dates", "TimeZones"] +git-tree-sha1 = "4af3552bf0cf4a071bf3d14bd20023ea70f31b62" +uuid = "0f7cfa37-7abf-4834-b969-a8aa512401c2" +version = "1.6.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.ZipArchives]] +deps = ["ArgCheck", "CodecInflate64", "CodecZlib", "InputBuffers", "PrecompileTools", "TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "83f728ecb873c58b794964f8b4bed811814d4b0d" +uuid = "49080126-0e18-4c2a-b176-c102e4b3760c" +version = "2.6.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.15.0+0" + +[[deps.libsodium_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026" +uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" +version = "1.0.21+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/app/Project.toml b/app/Project.toml new file mode 100644 index 0000000..c97970d --- /dev/null +++ b/app/Project.toml @@ -0,0 +1,19 @@ +[deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1" +NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" +Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" +PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" +ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" diff --git a/app/config.json b/app/config.json new file mode 100644 index 0000000..b2426bb --- /dev/null +++ b/app/config.json @@ -0,0 +1,39 @@ +{ + "configVersion": "0.1", + "mqttServerInfo": { + "description": "mqtt server", + "port": 1883, + "broker": "mqtt.yiem.cc" + }, + "natsServerInfo": { + "description": "nats server", + "port": 4222, + "broker": "nats.yiem.cc" + }, + "developmentStatus": { + "value": "testing", + "description": "agent status, could be testing or production" + }, + "servicetopic": { + "mqtttopic": [ + "/yiem/hq/agent/sommelier/backend/db/api_v1/testing" + ], + "description": "a topic this service are waiting for service request" + }, + "role": { + "value": "sommelier", + "description": "agent role" + }, + "organization": { + "value": "yiem", + "description": "organization name" + }, + "db": { + "description": "wine db", + "host": "192.168.88.12", + "port": "10201", + "dbname": "wineDB", + "user": "yiemtechnologies", + "password": "yiemtechnologies@Postgres_0.0" + } +} \ No newline at end of file diff --git a/app/env_preparation.jl b/app/env_preparation.jl new file mode 100644 index 0000000..e885414 --- /dev/null +++ b/app/env_preparation.jl @@ -0,0 +1,280 @@ +""" +For docker container: + - This script must be placed at the root of your project e.g. /appfolder/app + - This script expects all other files are in workfolder/temp e.g. /appfolder/app/temp +""" + +using Pkg + +programName = "someapp" # use my project name here +devPackageName = "somepackage" +appfolder = pwd() # use current folder as project folder + + +# ---------------------------------------------- 100 --------------------------------------------- # + +""" list of all recommend package +required_packages = [ + "Arrow" + "BenchmarkTools" + "CSV" + "CUDA" + "ClickHouse" + "DataFrames" + "DataFramesMeta" + "DataStructures" + "Distributions" + "Enzyme" + "FileIO" + "Flux" + "Genie" + "HTTP" + "ImageTransformations" + "Images" + "IterTools" + "JSON3" + "LinearAlgebra" + "Logging" + "Lux" + "MLDataUtils" + "MLDatasets" + "MLLabelUtils" + "Makie" + "ODBC" + "Optimisers" + "ParameterSchedulers" + "PkgTemplates" + "ProgressMeter" + "PyCall" + "Random" + "Revise" + "Serialization" + "Statistics" + "Stipple" + "TensorBoardLogger" + "TextAnalysis" + "Transformers" + "UUIDs" + "WordTokenizers" + "Zygote" + "https://github.com/denglerchr/Mosquitto.jl" +] +""" + + +# ---------------------------------------------------------------------------- # +# add julia packages in container's general registry folder # +# ---------------------------------------------------------------------------- # +privatejuliapkgpath = "$appfolder/dev" +required_julia_private_packages = [ #CHANGE + # "https://git.yiem.cc/ton/GeneralUtils", + # "https://git.yiem.cc/ton/LLMMCTS", + # "https://git.yiem.cc/ton/SQLLLM", + # "https://git.yiem.cc/ton/YiemAgent -b v0.1.1", +] + + +# ---------------------------------------------------------------------------- # +# add julia package in app folder # +# ---------------------------------------------------------------------------- # +required_julia_packages = [ #CHANGE + "Revise", + # "CondaPkg", + # "PythonCall", + "HTTP", + "Oxygen", + "JSON", + "Dates", + "UUIDs", + "Random", + "URIs", + "DataStructures", + "FileIO", + "PrettyPrinting", + "ZipArchives", + "LibPQ", + "ProgressMeter", + "NATS" +] # use only for new project # use only for new project + +function copy_folder_contents(src_folder::AbstractString, dst_folder::AbstractString) + # Get a list of all files in the source folder + files = readdir(src_folder) + + # Iterate over each file and copy it to the destination folder + for itemname in files + src_path = joinpath(src_folder, itemname) + dst_path = joinpath(dst_folder, itemname) + if isdir(src_path) + run(`cp -r $src_path $dst_path`) + else + run(`cp $src_path $dst_path`) + end + end +end + +function install_required_julia_packages(required_julia_packages) + for i in required_julia_packages + println("adding ", i) + if startswith(i, "http") + Pkg.add(url=i) + else + Pkg.add(i) + end + end +end + +function install_required_julia_private_packages(required_julia_private_packages::Vector, + appfolderpath::String, privatejuliapkgpath::String) + + !isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing + cd(privatejuliapkgpath) + + if length(required_julia_private_packages) > 0 + # remove private packages already in Project.toml in case private package path is different + for i in required_julia_private_packages + _pkgname = split(i, "/")[end] + _pkgname = split(_pkgname, " ")[1] + pkgname = split(_pkgname, ".")[1] + println("removing Julia private package: $pkgname if it already in Project.toml") + try Pkg.rm(pkgname) catch end + end + + # clone all required julia private packages + for i in required_julia_private_packages + _pkgname = split(i, "/")[end] + _pkgname = split(_pkgname, " ")[1] + pkgname = split(_pkgname, ".")[1] + pkgpath = joinpath(privatejuliapkgpath, pkgname) + println("cloning Julia private package: $pkgname") + + # only clone a package if it is not found in local + gitcommand = "git clone $i" + args = split(gitcommand, " ") + !isdir(pkgpath) ? run(`$args`) : nothing + end + + # install all required julia private packages + for i in required_julia_private_packages + _pkgname = split(i, "/")[end] + _pkgname = split(_pkgname, " ")[1] + pkgname = split(_pkgname, ".")[1] # in case there is a .jl at the end of package name + pkgpath = joinpath(privatejuliapkgpath, pkgname) + println("installing Julia private package: $pkgname") + + Pkg.develop(path=pkgpath) + end + end + + cd(appfolderpath) +end + + +# ------------------------------------------------------------------------------------------------ # +# start script # +# ------------------------------------------------------------------------------------------------ # + +# all files are put in ./temp (most likely in a docker container) +if isdir(joinpath(appfolder, "temp")) + if isfile(joinpath(appfolder, "temp", "Project.toml")) # use this option if one already have project.toml + copy_folder_contents(joinpath(appfolder, "temp"), appfolder) + Pkg.activate(".") + install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath) + Pkg.instantiate() + else # no Project.toml, create new package + copy_folder_contents(joinpath(appfolder, "temp"), appfolder) + Pkg.activate(".") + length(required_julia_packages) != 0 ? install_required_julia_packages(required_julia_packages) : nothing + + # for app + write("main.jl", "using Revise # remove when this package is completed\n", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n") + mkdir("testapp") + cd("testapp") + write("runtests.jl", "# ---------------------------------------------- 100 --------------------------------------------- #\n") + cd(appfolder) + install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath) + + # create new dev package folder + !isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing + cd(privatejuliapkgpath) + Pkg.generate(devPackageName) + newPackagePath = joinpath(privatejuliapkgpath, devPackageName) + Pkg.develop(path=newPackagePath) + cd(newPackagePath) + mkdir("test") + cd("./test") + write("runtests.jl", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n") + cd(appfolder) + end +else # all files are in the current work folder + if isfile(joinpath(appfolder, "Project.toml")) # use this option if one already have project.toml + Pkg.activate(".") + install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath) + Pkg.instantiate() + else # create new package folder + Pkg.activate(".") + length(required_julia_packages) != 0 ? install_required_julia_packages(required_julia_packages) : nothing + + # for app + write("main.jl", "using Revise # remove when this package is completed\n", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n") + mkdir("testapp") + cd("testapp") + write("runtests.jl", "# ---------------------------------------------- 100 --------------------------------------------- #\n") + cd(appfolder) + install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath) + + # create new dev package folder + !isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing + cd(privatejuliapkgpath) + Pkg.generate(devPackageName) + newPackagePath = joinpath(privatejuliapkgpath, devPackageName) + Pkg.develop(path=newPackagePath) + cd(newPackagePath) + mkdir("test") + cd("./test") + write("runtests.jl", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n") + cd(appfolder) + + # preparing typical julia project finished now doing project's specific preparation here + end +end + + +# ---------------------------------------------------------------------------- # +# python libraries used by Julia's PythonCall # +# ---------------------------------------------------------------------------- # +pythonlib = [] +channels = ["anaconda", "conda-forge", "pytorch"] +if length(pythonlib) != 0 + using CondaPkg; + for i in channels CondaPkg.add_channel(i) end + for i in pythonlib CondaPkg.add_pip(i) end +end + +# ---------------------------------------------------------------------------- # +# create symlink # +# ---------------------------------------------------------------------------- # +# if isfile("/app/run.jl") +# symlink("/app/$programName/run.jl", "/app/run.jl") +# elseif isfile("/app/$programName/main.py") +# symlink("/app/$programName/main.py", "/app/main.py") +# else +# cd("/app") +# write("no run.jl in package folder.txt", "") +# end + + +# ------------------------------------------------------------------------------------------------ # +# for web app # +# ------------------------------------------------------------------------------------------------ # +# intendedURL = "wine.yiem.cc/hq/agent/sommelier/frontend/dbadmin" # e.g. "wine.yiem.cc/hq/agent/sommelier/frontend/dbadmin" +# urlList = split(intendedURL, "/")[2:end] # e.g. ["hq", "agent", "sommelier", "frontend", "dbadmin"] +# # use joinpath to compose the path from urlList +# _path = joinpath(urlList...) +# path = "./$_path" +# mkpath(path) +# # move all files in the current folder to the path +# copy_folder_contents(joinpath(appfolder, "temp"), path) + + +println("--> Julia env preparation done") \ No newline at end of file diff --git a/app/etc.jl b/app/etc.jl new file mode 100644 index 0000000..9f8df1f --- /dev/null +++ b/app/etc.jl @@ -0,0 +1,103 @@ +curl http://localhost:8080/v1/audio/speech -H "Content-Type: application/json" -d '{ + "model": "tts-1", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy" +}' --output speech.mp3 + + + + + + + + +curl http://localhost:8080/v1/images/generations -H "Content-Type: application/json" -d '{ + "prompt": "floating hair, portrait, ((loli)), ((one girl)), cute face, hidden hands, asymmetrical bangs, beautiful detailed eyes, eye shadow, hair ornament, ribbons, bowties, buttons, pleated skirt, (((masterpiece))), ((best quality)), colorful|((part of the head)), ((((mutated hands and fingers)))), deformed, blurry, bad anatomy, disfigured, poorly drawn face, mutation, mutated, extra limb, ugly, poorly drawn hands, missing limb, blurry, floating limbs, disconnected limbs, malformed hands, blur, out of focus, long neck, long body, Octane renderer, lowres, bad anatomy, bad hands, text", + "model": "animagine-xl", + "step": 51, + "size": "1024x1024" + }' + + + + + + +curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ + "model": "gpt-4-vision-preview", + "messages": [ + { + "role": "user", "content": [ + {"type":"text", "text": "What is in the image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + } + ], + "temperature": 0.9 + } + ] + }' + + + +""" correct insert statement +INSERT INTO wine (grape, additional_search_term, acidity, updated_time, country, description, region, tannin, winery, intensity, sweetness, tasting_notes, wine_name, wine_id, wine_type, other_attributes, fizziness, serving_temperature, created_time) VALUES ('new', ARRAY['text1', 'text2'], '0', '2024-10-01T12:08:46.695+00:00', 'new', 'new', 'new', '0', 'ddd', '0', '0', 'new', 'new_wine', 'e1597803-ab5e-4653-935c-38916e2e0827', 'new', '{"attribute": "sometext"}', '0', '0', '2024-10-01T12:08:46.695+00:00') +""" + + +data = Dict{Symbol, Any}(:additional_search_term => ["NA1", "NA2"], :acidity => 0, :country => "NA", :description => "NA", :region => "NA", :intensity => 0, :wine_name => "new_wine", :fizziness => 0, :tannin => 0, :winery => "NA", :sweetness => 0, :tasting_notes => "NA", :wine_type => "NA", :other_attributes => Dict{Symbol, Any}(:attribute1 => "sometext", :attribute2 => 0), :serving_temperature => 0, :grape => "NA") + + + + +I have a Postgres table created with this SQL: +CREATE TABLE wine (wine_id uuid, intensity integer, sweetness integer, tannin integer, acidity integer, fizziness integer, other_attributes jsonb, created_time timestamp with time zone, updated_time timestamp with time zone, description text, tasting_notes text, wine_name character varying, winery character varying, region character varying, country character varying, wine_type character varying, grape character varying, serving_temperature character varying, additional_search_term ARRAY); + +Here are the data for updating: + update_data = Dict{Symbol, Any}(:additional_search_term => ["NA1", "NA2"], :acidity => "0", :country => "NA", :description => "NA", :region => "NA", :intensity => "0", :wine_name => "new_wine", :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad", :fizziness => "0", :tannin => "0", :winery => "ccc", :sweetness => "0", :tasting_notes => "NA", :wine_type => "NA", :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}", :serving_temperature => "0", :grape => "NA") + +Write a Julia function to generate SQL statement to update the table. + I will provide the following as arguments: + 1) table name + 2) a dictionary for update data with symbol keys + 3) the list of keys in the dictionary that are the table id. + + I also need a Julia function to insert data into the table. + +P.S. 1) do not use a comprehension. 2) Postgres jsonb column requires this form: '{"key1": value, "key2": "text", ...}' + + + + +Here is an example of a Julia dictionary: +insert_data = Dict{Symbol, Any}(:additional_search_term => ["NA1", "NA2"], :acidity => 0, :country => "NA", :description => "NA", :region => "NA", :intensity => 0, :wine_name => "new_wine", :fizziness => 0, :tannin => 0, :winery => "NA", :sweetness => 0, :tasting_notes => "NA", :wine_type => "NA", :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}", :serving_temperature => 0, :grape => "NA") + +I also need a Julia function to insert data into the table. + +P.S. 1) do not use a comprehension. 2) Postgres jsonb column requires this form: '{"key1": value, "key2": "text", ...}' + + + + + + + +d = Dict( + :attribute1 => "hello world", + :attribute2 => 555, + :attribute3 => Dict( + :attribute4 => "text", + :attribute5 => 666 + ) +) + +# println hello world +println(d[:attribute1]) + + +'{"attribute3":{"attribute5":666,"attribute4":"text"},"attribute1":"hello world","attribute2":555}' + + diff --git a/app/main backup 2.jl b/app/main backup 2.jl new file mode 100644 index 0000000..35367ae --- /dev/null +++ b/app/main backup 2.jl @@ -0,0 +1,744 @@ + +using JSON3, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames +using GeneralUtils +using Base.Threads + +# ---------------------------------------------- 100 --------------------------------------------- # + + +""" Expected incomming MQTT message format for this service: + { + "msgMeta": { + "msgPurpose": "updateStatus", + "requestresponse": "request", + "timestamp": "2024-03-29T05:8:48.362", + "replyToMsgId": null, + "receiverId": null, + "getpost": "get", + "msgId": "e5c09bd8-7100-4e4e-bb43-05bee589a22c", + "acknowledgestatus": null, + "sendTopic": "/agent/wine/backend/chat/api/v1/prompt", + "receiverName": "agent-wine-backend", + "replyTopic": "/agent/wine/frontend/chat/api/v1/txt/receive", + "senderName": "agent-wine-frontend-chat", + "senderId": "0938a757-e0ee-40a9-8355-5e24906a87cd" + }, + "payload" : { + "text": "hello" + } + + } +""" + + + + + +# load config +config = copy(JSON3.read("../mountvolume/config.json")) + +function executeSQL(sql::T) where {T<:AbstractString} + DBconnection = LibPQ.Connection("host=$(config[:db][:host]) port=$(config[:db][:port]) dbname=$(config[:db][:dbname]) user=$(config[:db][:user]) password=$(config[:db][:password])") + result = LibPQ.execute(DBconnection, sql) + LibPQ.close(DBconnection) + return result +end + +function listAllTableColumns(tablename::String)::Vector + sql = + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = '$tablename'; + """ + + response = executeSQL(sql) + df = DataFrame(response) + return Symbol.(df[:, 1]) +end + + +function load_winetable(args::Dict) + tablename = "wine" + sql = + """ + SELECT * + FROM $tablename; + """ + + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + return df + else + return nothing + end +end + +function insert_masterWineDB(args::Dict) + tablename = "wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateInsertSQL(tablename, columnToUpdate, args) + println("insert_masterWineDB() SQL: $sql") + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function delete_masterWineDB(args::Dict) + tablename = "wine" + + sql = + """ + DELETE FROM $tablename + WHERE wine_id = '$(args[:wine_id])'; + """ + println("delete_masterWineDB() SQL: $sql") + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function edit_masterWineDB(args::Dict) + tablename = "wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateUpdateSQL(tablename, columnToUpdate, args, [:wine_id]) + println("") + println("edit_masterWineDB() SQL: $sql") + + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function search_masterWineTable(args::Dict) + tablename = "wine" + _searchkeyword = args[:searchkeyword] + searchkeyword1 = split(_searchkeyword, "'") # postgres not support search keyword containing ' + searchkeyword_length = length.(searchkeyword1) + _, searchkeywordIndex, _ = GeneralUtils.findMax(searchkeyword_length) + searchkeyword = searchkeyword1[searchkeywordIndex] + + columnname = args[:searchcolumn] + + # check parameters + if searchkeyword == "" + println("Error, search keyword is empty") + return "Error, search keyword is empty" + elseif columnname == "" + println("Error, search column name is empty") + return "Error, search column name is empty" + elseif isa(columnname, Number) + println("Error, search search column name must be string") + return "Error, search search column name must be string" + end + + sql = + if searchkeyword == "*" + """ + SELECT * + FROM $tablename; + """ + else + """ + SELECT * + FROM $tablename + WHERE $columnname ILIKE '%$searchkeyword%' LIMIT 1000; + """ + end + println("~~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function load_retailerWineInventory(args::Dict) + retailer_id = args[:retailerid] + if length(retailer_id) == 0 + println("No retailer_id provided ", @__FILE__, " ", @__LINE__) + return nothing + end + sql = + """ + SELECT + w.wine_id, + w.winery, + w.wine_name, + w.vintage, + w.grape, + w.wine_type, + w.region, + w.country, + w.created_time, + w.updated_time, + rw.price, + rw.currency + FROM + wine w + JOIN + retailer_wine rw ON w.wine_id = rw.wine_id + WHERE + rw.retailer_id = '$retailer_id'; + """ + # sql = + # """ + # SELECT + # w.wine_id, + # w.wine_name, + # w.winery, + # w.region, + # w.country, + # w.wine_type, + # w.grape, + # w.serving_temperature, + # w.intensity, + # w.sweetness, + # w.tannin, + # w.acidity, + # w.fizziness, + # w.tasting_notes, + # w.note, + # w.other_attributes, + # w.created_time, + # w.updated_time, + # w.description, + # rw.vintage, + # rw.price, + # rw.currency + # FROM + # wine w + # JOIN + # retailer_wine rw ON w.wine_id = rw.wine_id + # WHERE + # rw.retailer_id = '$retailer_id'; + # """ + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + return df + else + return nothing + end +end + +function insert_retailerWineInventory(args::Dict) + tablename = "retailer_wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateInsertSQL(tablename, columnToUpdate, args) + + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function delete_retailerWineInventory(args::Dict) + tablename = "retailer_wine" + + sql = + """ + DELETE FROM $tablename + WHERE retailer_id = '$(args[:retailer_id])' AND wine_id = '$(args[:wine_id])'; + """ + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function edit_retailerWineInventory(args::Dict) + # result = delete_retailerWineInventory(args) + # result = insert_retailerWineInventory(args) + + tablename = "retailer_wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateUpdateSQL(tablename, columnToUpdate, args, [:retailer_id, :wine_id]) + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + + + +# ---------------------------------------------- 100 --------------------------------------------- # +function runServiceInstance( + receiveUserMsgChannel::Channel, + outputchannel::Channel, + config::Dict, + timeout::Int64, + ) + workDict = Dict() + latestUserMsgTimeStamp::DateTime = Dates.now() + + while true + # check for new user message + if isready(receiveUserMsgChannel) + incomingMsg = take!(receiveUserMsgChannel) + incomingPayload = incomingMsg[:payload] + latestUserMsgTimeStamp = Dates.now() + println("") + println("<-- incomingMsg ", @__FILE__, " ", @__LINE__) + println(incomingMsg) + + # sending msg back to sender i.e. LINE + msgMeta = GeneralUtils.generate_msgMeta( + incomingMsg[:msgMeta][:replyTopic]; + senderName = "wine_assistant_backend_db", + senderId= GeneralUtils.uuid4snakecase(), + replyToMsgId= incomingMsg[:msgMeta][:msgId], + mqttBrokerAddress= config[:mqttServerInfo][:broker], + mqttBrokerPort= config[:mqttServerInfo][:port], + ) + + #[WORKING] add other DB call function here + if incomingPayload[:functioncall] == "search_masterWineTable" + result = GeneralUtils.timeout(search_masterWineTable, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + # println("") + # println("-~~~ outgoingMsg ", @__FILE__, " ", @__LINE__) + # pprint(outgoingMsg) + + elseif incomingPayload[:functioncall] == "load_retailerWineInventory" + println("load_retailerWineInventory()") + # incomingMsg is requesting metadata by not having :dataTransferSessionID + if !haskey(incomingPayload, :dataTransferSessionID) + + # load data + df = load_retailerWineInventory(incomingPayload[:args]) + vd = GeneralUtils.dfToVectorDict(df) + disvd = GeneralUtils.disintegrate_vectorDict(vd, 100) + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg; data=disvd) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + + # merge dictionary so that keys are on the same level + for (k, v) in result + outgoingMsg[:payload][k] = v + end + + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + else + + # call dataTransferOverMQTT_sender + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + for (k, v) in result + outgoingMsg[:payload][k] = v + end + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + end + + # elseif incomingPayload[:functioncall] == "load_retailerWineInventory" + # result = GeneralUtils.timeout(load_retailerWineInventory, 30; + # fargs=incomingPayload[:args]) + + # outgoingMsg = Dict( + # :msgMeta=> msgMeta, + # :payload=> Dict( + # :functioncall=> incomingPayload[:functioncall], + # :result=> result + # ) + # ) + # _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "insert_retailerWineInventory" + println("insert_retailerWineInventory()") + result = GeneralUtils.timeout(insert_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "delete_retailerWineInventory" + println("delete_retailerWineInventory()") + retailer_id = incomingPayload[:args][:retailer_id] + wine_id = incomingPayload[:args][:wine_id] + if length(retailer_id) != 0 && length(wine_id) != 0 + result = GeneralUtils.timeout(delete_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + else + println("Skipped call, insufficient args for delete_retailerWineInventory() retailer_id: $retailer_id wine_id: $wine_id") + end + + elseif incomingPayload[:functioncall] == "edit_retailerWineInventory" + println("edit_retailerWineInventory()") + retailer_id = incomingPayload[:args][:retailer_id] + wine_id = incomingPayload[:args][:wine_id] + if length(retailer_id) != 0 && length(wine_id) != 0 + result = GeneralUtils.timeout(edit_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + else + println("Skipped call, insufficient args for edit_retailerWineInventory() retailer_id: $retailer_id wine_id: $wine_id") + end + + elseif incomingPayload[:functioncall] == "load_winetable" + println("load_winetable()") + # incomingMsg is requesting metadata by not having :dataTransferSessionID + if !haskey(incomingPayload, :dataTransferSessionID) + # load data + df = load_winetable(incomingPayload[:args]) + vd = GeneralUtils.dfToVectorDict(df) + println(typeof(vd)) + disvd = GeneralUtils.disintegrate_vectorDict(vd, 100) + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg; data=disvd) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + + for (k, v) in result + outgoingMsg[:payload][k] = v + end + + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + pprintln(outgoingMsg) + + else + + # call dataTransferOverMQTT_sender + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + for (k, v) in result + outgoingMsg[:payload][k] = v + end + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + end + + elseif incomingPayload[:functioncall] == "insert_masterWineDB" + println("insert_masterWineDB()") + result = GeneralUtils.timeout(insert_masterWineDB, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "delete_masterWineDB" + result = GeneralUtils.timeout(delete_masterWineDB, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "edit_masterWineDB" + println("edit_masterWineDB()") + result = GeneralUtils.timeout(edit_masterWineDB, 30; + fargs=incomingPayload[:args]) + # result = edit_masterWineDB(incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + else + println("") + println("~~~ The requested function is not defined ", @__FILE__, " ", @__LINE__) + end + end + + # self terminate if too long inactivity + timediff = GeneralUtils.timedifference(latestUserMsgTimeStamp, Dates.now(), "minutes") + if timediff > timeout + result = Dict(:exitreason=>"timeout", :timestamp=>Dates.now()) + put!(outputchannel, result) + disconnect(a.mqttClient) + break + else + sleep(1) # allowing on_msg_2, asyncmove above and other process to run + end + end +end + +sessionDict = Dict{String,Any}() +mqttMsgReceiveChannel = (ch1=Channel(8),) # store msg that coming into servicetopic +keepaliveChannel::Channel{Dict} = Channel{Dict}(8) + +# Define the callback for receiving messages. +function onMsgCallback_1(topic, payload) + jobj = JSON3.read(String(payload)) + incomingMqttMsg = copy(jobj) # convert json object into julia dictionary recursively + + if occursin("db", topic) + # println("~~~ incomingMqttMsg ", incomingMqttMsg) + put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg) + elseif occursin("keepalive", topic) + put!(keepaliveChannel, incomingMqttMsg) + else + println("undefined condition ", @__FILE__, " ", @__LINE__) + end +end + +mqttInstance = GeneralUtils.mqttClientInstance_v2( + config[:mqttServerInfo][:broker], + config[:servicetopic][:mqtttopic], + mqttMsgReceiveChannel, + keepaliveChannel, + onMsgCallback_1 +) + +println("ready!") + +# ------------------------------------------------------------------------------------------------ # +# this service main loop # +# ------------------------------------------------------------------------------------------------ # +function main() + sessiontimeout = 1*1*10 # timeout in minutes + checkSessionTimeout = 10 # minutes + clearedSessionTimestamp = Dates.now() + lastMsgId = nothing + while true + + # check if mqtt connection is still up + _ = GeneralUtils.checkMqttConnection!(mqttInstance; keepaliveCheckInterval=30) + + # check for new session + if isready(mqttMsgReceiveChannel[:ch1]) + msg = popfirst!(mqttMsgReceiveChannel[:ch1]) + # println("~~~ new msg ", msg[:payload]) + + # @spawn new runAgentInstance and store it in sessionDict + # use agent's frontend id because 1 backend agent per 1 frontend session + sessionId = msg[:msgMeta][:senderId] + sessionId = replace(sessionId, "-" => "_") # julia can't use "-" in a dict key + msgId = msg[:msgMeta][:msgId] + + if msgId != lastMsgId && sessionId ∉ keys(sessionDict) + lastMsgId = msgId + inputch = Channel{Dict}(8) + outputch = Channel{Dict}(8) + process = @spawn runServiceInstance(inputch, outputch, config, sessiontimeout) + # process = runServiceInstance(inputch, outputch, config, sessiontimeout) #XXX use spawn version + println("~~ instantiate session success") + + # call runAgentInstance() and store it in sessionDict to be able to check on it later + sessionDict[sessionId] = Dict( + :inputchannel=> inputch, + :outputchannel=> outputch, + :process=> process, + ) + + put!(sessionDict[sessionId][:inputchannel], msg) + else + + put!(sessionDict[sessionId][:inputchannel], msg) + end + end + + # check for process completed msg in serviceInternalTopic and delete it from sessionDict + # self terminate if too long inactivity + timediff = GeneralUtils.timedifference(clearedSessionTimestamp, Dates.now(), "minutes") + if timediff > checkSessionTimeout + for (sessionId, v) in sessionDict + if isready(v[:outputchannel]) + result = take!(v[:outputchannel]) + if result[:exitreason] == "timeout" + println("sessionId $(sessionId) has been deleted because it is timed out") + delete!(sessionDict, sessionId) + end + end + end + clearedSessionTimestamp = Dates.now() + end + + # sleep is needed because MQTTClient use async. while true loop leave no + # chance for control to switch to on_msg() + sleep(1) + end +end + +main() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/main backup.jl b/app/main backup.jl new file mode 100644 index 0000000..f770d4f --- /dev/null +++ b/app/main backup.jl @@ -0,0 +1,841 @@ + +using JSON3, MQTTClient, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames +using GeneralUtils +using Base.Threads + +# ---------------------------------------------- 100 --------------------------------------------- # + + +""" Expected incomming MQTT message format for this service: + { + "msgMeta": { + "msgPurpose": "updateStatus", + "requestresponse": "request", + "timestamp": "2024-03-29T05:8:48.362", + "replyToMsgId": null, + "receiverId": null, + "getpost": "get", + "msgId": "e5c09bd8-7100-4e4e-bb43-05bee589a22c", + "acknowledgestatus": null, + "sendTopic": "/agent/wine/backend/chat/api/v1/prompt", + "receiverName": "agent-wine-backend", + "replyTopic": "/agent/wine/frontend/chat/api/v1/txt/receive", + "senderName": "agent-wine-frontend-chat", + "senderId": "0938a757-e0ee-40a9-8355-5e24906a87cd" + }, + "payload" : { + "text": "hello" + } + + } +""" + + + + + +# load config +config = copy(JSON3.read("config.json")) + +function executeSQL(sql::T) where {T<:AbstractString} + DBconnection = LibPQ.Connection("host=192.168.88.12 port=10201 dbname=wineDB user=yiemtechnologies password=yiemtechnologies@Postgres_0.0") + result = LibPQ.execute(DBconnection, sql) + LibPQ.close(DBconnection) + return result +end + +# function executeSQLVectorDB(sql) +# DBconnection = LibPQ.Connection("host=192.168.88.12 port=5433 dbname=SQLVectorDB user=yiemtechnologies@gmail.com password=yiem@Postgres_0.0") +# result = LibPQ.execute(DBconnection, sql) +# LibPQ.close(DBconnection) +# return result +# end + +# function addSQLVectorDB(state) +# # get embedding of the query +# query = [state[:thoughtHistory][:question]] +# msgMeta = GeneralUtils.generate_msgMeta( +# config[:externalservice][:text2textinstruct][:mqtttopic]; +# msgPurpose= "embedding", +# senderName= "yiemagent", +# senderId= string(uuid4()), +# receiverName= "text2textinstruct", +# mqttBrokerAddress= config[:mqttServerInfo][:broker], +# mqttBrokerPort= config[:mqttServerInfo][:port], +# ) + +# outgoingMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict( +# :text=> query +# ) +# ) +# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) +# embedding = response[:response][:embeddings][1] + +# # check whether there is close enough vector already store in vectorDB. if no, add, else skip +# sql = +# """ +# SELECT *, embedding <-> '$embedding' as distance +# FROM sql_statement_repository +# ORDER BY distance LIMIT 1; +# """ +# response = executeSQLVectorDB(sql) +# df = DataFrame(response) +# row, col = size(df) +# distance = row == 0 ? Inf : df[1, :distance] +# if row == 0 || distance > 1 # no close enough SQL stored in the database +# latestKey, _ = GeneralUtils.findHighestIndexKey(state[:thoughtHistory], :action_input) +# _sqlStatement = state[:thoughtHistory][latestKey] +# if occursin("SELECT", _sqlStatement) # make sure it is an SQL statement before adding into DB +# sqlStatementBase64 = base64encode(_sqlStatement) +# sqlStatement = replace(_sqlStatement, "'"=>"") +# sql = +# """ +# INSERT INTO sql_statement_repository (question, sql_statement, sql_statement_base64, embedding) VALUES ('$query', '$sqlStatement', '$sqlStatementBase64', '$embedding'); +# """ +# _ = executeSQLVectorDB(sql) +# println("~~ added new SQL statement to vectorDB ", @__FILE__, " ", @__LINE__) +# println(sqlStatement) +# end +# end +# end + +# function querySQLVectorDB(state) + +# # provide similarSQL at the first time thinking only +# latestKey, _ = GeneralUtils.findHighestIndexKey(state[:thoughtHistory], :action_input) +# if latestKey === nothing +# # get embedding of the query +# query = [state[:thoughtHistory][:question]] +# msgMeta = GeneralUtils.generate_msgMeta( +# config[:externalservice][:text2textinstruct][:mqtttopic]; +# msgPurpose= "embedding", +# senderName= "yiemagent", +# senderId= string(uuid4()), +# receiverName= "text2textinstruct", +# mqttBrokerAddress= config[:mqttServerInfo][:broker], +# mqttBrokerPort= config[:mqttServerInfo][:port], +# ) + +# outgoingMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict( +# :text=> query +# ) +# ) +# response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) +# embedding = response[:response][:embeddings][1] + +# # check whether there is close enough vector already store in vectorDB. if no, add, else skip +# sql = +# """ +# SELECT *, embedding <-> '$embedding' as distance +# FROM sql_statement_repository +# ORDER BY distance LIMIT 1; +# """ +# response = executeSQLVectorDB(sql) +# df = DataFrame(response) +# row, col = size(df) +# distance = row == 0 ? Inf : df[1, :distance] +# if row != 0 && distance < 100 +# # if there is usable SQL, return it. +# sqlStatementBase64 = df[1, :sql_statement_base64] +# sqlStatement = String(base64decode(sqlStatementBase64)) +# println("~~~ getting SQL statement from vectorDB ", @__FILE__, " ", @__LINE__) +# println(sqlStatement) +# return sqlStatement +# else +# return nothing +# end +# end +# return nothing +# end + + +function listAllTableColumns(tablename::String) + sql = + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = '$tablename'; + """ + + response = executeSQL(sql) + df = DataFrame(response) + return Symbol.(df[:, 1]) +end + + +function load_winetable(args::Dict) + tablename = "wine" + sql = + """ + SELECT * + FROM $tablename; + """ + + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + return df + else + return nothing + end +end + +function insert_masterWineDB(args::Dict) + tablename = "wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateInsertSQL(tablename, columnToUpdate, args) + println("insert_masterWineDB() SQL: $sql") + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function delete_masterWineDB(args::Dict) + tablename = "wine" + + sql = + """ + DELETE FROM $tablename + WHERE wine_id = '$(args[:wine_id])'; + """ + println("delete_masterWineDB() SQL: $sql") + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function edit_masterWineDB(args::Dict) + tablename = "wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateUpdateSQL(tablename, columnToUpdate, args, [:wine_id]) + println("") + println("edit_masterWineDB() SQL: $sql") + + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function search_masterWineTable(args::Dict) + tablename = "wine" + _searchkeyword = args[:searchkeyword] + searchkeyword1 = split(_searchkeyword, "'") # postgres not support search keyword containing ' + searchkeyword_length = length.(searchkeyword1) + _, searchkeywordIndex, _ = GeneralUtils.findMax(searchkeyword_length) + searchkeyword = searchkeyword1[searchkeywordIndex] + + columnname = args[:searchcolumn] + sql = + if searchkeyword == "*" + """ + SELECT * + FROM $tablename; + """ + else + """ + SELECT * + FROM $tablename + WHERE $columnname ILIKE '%$searchkeyword%' LIMIT 100; + """ + end + println("~~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function load_retailerWineInventory(args::Dict) + retailer_id = args[:retailerid] + if length(retailer_id) == 0 + println("No retailer_id provided ", @__FILE__, " ", @__LINE__) + return nothing + end + sql = + """ + SELECT + w.wine_id, + w.winery, + w.wine_name, + w.vintage, + w.grape, + w.wine_type, + w.region, + w.country, + w.created_time, + w.updated_time, + rw.price, + rw.currency + FROM + wine w + JOIN + retailer_wine rw ON w.wine_id = rw.wine_id + WHERE + rw.retailer_id = '$retailer_id'; + """ + # sql = + # """ + # SELECT + # w.wine_id, + # w.wine_name, + # w.winery, + # w.region, + # w.country, + # w.wine_type, + # w.grape, + # w.serving_temperature, + # w.intensity, + # w.sweetness, + # w.tannin, + # w.acidity, + # w.fizziness, + # w.tasting_notes, + # w.note, + # w.other_attributes, + # w.created_time, + # w.updated_time, + # w.description, + # rw.vintage, + # rw.price, + # rw.currency + # FROM + # wine w + # JOIN + # retailer_wine rw ON w.wine_id = rw.wine_id + # WHERE + # rw.retailer_id = '$retailer_id'; + # """ + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + return df + else + return nothing + end +end + +function insert_retailerWineInventory(args::Dict) + tablename = "retailer_wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateInsertSQL(tablename, columnToUpdate, args) + + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function delete_retailerWineInventory(args::Dict) + tablename = "retailer_wine" + + sql = + """ + DELETE FROM $tablename + WHERE retailer_id = '$(args[:retailer_id])' AND wine_id = '$(args[:wine_id])'; + """ + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function edit_retailerWineInventory(args::Dict) + # result = delete_retailerWineInventory(args) + # result = insert_retailerWineInventory(args) + + tablename = "retailer_wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateUpdateSQL(tablename, columnToUpdate, args, [:retailer_id, :wine_id]) + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + + + +# ---------------------------------------------- 100 --------------------------------------------- # +function runServiceInstance( + receiveUserMsgChannel::Channel, + outputchannel::Channel, + config::Dict, + timeout::Int64, + ) + workDict = Dict() + latestUserMsgTimeStamp::DateTime = Dates.now() + + while true + # check for new user message + if isready(receiveUserMsgChannel) + incomingMsg = take!(receiveUserMsgChannel) + incomingPayload = incomingMsg[:payload] + latestUserMsgTimeStamp = Dates.now() + println("") + println("<-- incomingMsg ", @__FILE__, " ", @__LINE__) + println(incomingMsg) + + # sending msg back to sender i.e. LINE + msgMeta = GeneralUtils.generate_msgMeta( + incomingMsg[:msgMeta][:replyTopic]; + senderName = "wine_assistant_backend_db", + senderId= GeneralUtils.uuid4snakecase(), + replyToMsgId= incomingMsg[:msgMeta][:msgId], + mqttBrokerAddress= config[:mqttServerInfo][:broker], + mqttBrokerPort= config[:mqttServerInfo][:port], + ) + + # add other DB call function here + if incomingPayload[:functioncall] == "search_masterWineTable" + println("search_masterWineTable()") + + result = GeneralUtils.timeout(search_masterWineTable, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + # println("") + # println("-~~~ outgoingMsg ", @__FILE__, " ", @__LINE__) + # pprint(outgoingMsg) + + elseif incomingPayload[:functioncall] == "load_retailerWineInventory" + println("load_retailerWineInventory()") + # incomingMsg is requesting metadata by not having :dataTransferSessionID + if !haskey(incomingPayload, :dataTransferSessionID) + + # load data + df = load_retailerWineInventory(incomingPayload[:args]) + vd = GeneralUtils.dfToVectorDict(df) + disvd = GeneralUtils.disintegrate_vectorDict(vd, 100) + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg; data=disvd) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + + for (k, v) in result + outgoingMsg[:payload][k] = v + end + + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + else + + # call dataTransferOverMQTT_sender + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + for (k, v) in result + outgoingMsg[:payload][k] = v + end + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + end + + # elseif incomingPayload[:functioncall] == "load_retailerWineInventory" + # result = GeneralUtils.timeout(load_retailerWineInventory, 30; + # fargs=incomingPayload[:args]) + + # outgoingMsg = Dict( + # :msgMeta=> msgMeta, + # :payload=> Dict( + # :functioncall=> incomingPayload[:functioncall], + # :result=> result + # ) + # ) + # _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "insert_retailerWineInventory" + println("insert_retailerWineInventory()") + result = GeneralUtils.timeout(insert_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "delete_retailerWineInventory" + println("delete_retailerWineInventory()") + retailer_id = incomingPayload[:args][:retailer_id] + wine_id = incomingPayload[:args][:wine_id] + if length(retailer_id) != 0 && length(wine_id) != 0 + result = GeneralUtils.timeout(delete_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + else + println("Skipped call, insufficient args for delete_retailerWineInventory() retailer_id: $retailer_id wine_id: $wine_id") + end + + elseif incomingPayload[:functioncall] == "edit_retailerWineInventory" + println("edit_retailerWineInventory()") + retailer_id = incomingPayload[:args][:retailer_id] + wine_id = incomingPayload[:args][:wine_id] + if length(retailer_id) != 0 && length(wine_id) != 0 + result = GeneralUtils.timeout(edit_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + else + println("Skipped call, insufficient args for edit_retailerWineInventory() retailer_id: $retailer_id wine_id: $wine_id") + end + + elseif incomingPayload[:functioncall] == "load_winetable" + println("load_winetable()") + # incomingMsg is requesting metadata by not having :dataTransferSessionID + if !haskey(incomingPayload, :dataTransferSessionID) + # load data + df = load_winetable(incomingPayload[:args]) + vd = GeneralUtils.dfToVectorDict(df) + println(typeof(vd)) + disvd = GeneralUtils.disintegrate_vectorDict(vd, 100) + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg; data=disvd) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + + for (k, v) in result + outgoingMsg[:payload][k] = v + end + + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + pprintln(outgoingMsg) + + else + + # call dataTransferOverMQTT_sender + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + for (k, v) in result + outgoingMsg[:payload][k] = v + end + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + end + + elseif incomingPayload[:functioncall] == "insert_masterWineDB" + println("insert_masterWineDB()") + result = GeneralUtils.timeout(insert_masterWineDB, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "delete_masterWineDB" + result = GeneralUtils.timeout(delete_masterWineDB, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "edit_masterWineDB" + println("edit_masterWineDB()") + result = GeneralUtils.timeout(edit_masterWineDB, 30; + fargs=incomingPayload[:args]) + # result = edit_masterWineDB(incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + else + println("") + println("~~~ The requested function is not defined ", @__FILE__, " ", @__LINE__) + end + end + + # self terminate if too long inactivity + timediff = GeneralUtils.timedifference(latestUserMsgTimeStamp, Dates.now(), "minutes") + if timediff > timeout + result = Dict(:exitreason=>"timeout", :timestamp=>Dates.now()) + put!(outputchannel, result) + disconnect(a.mqttClient) + break + else + sleep(1) # allowing on_msg_2, asyncmove above and other process to run + end + end +end + +sessionDict = Dict{String,Any}() +mqttMsgReceiveChannel = (ch1=Channel(8),) # store msg that coming into servicetopic +keepaliveChannel::Channel{Dict} = Channel{Dict}(8) + +# Define the callback for receiving messages. +function onMsgCallback_1(topic, payload) + jobj = JSON3.read(String(payload)) + incomingMqttMsg = copy(jobj) # convert json object into julia dictionary recursively + + if occursin("db", topic) + # println("~~~ incomingMqttMsg ", incomingMqttMsg) + put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg) + elseif occursin("keepalive", topic) + put!(keepaliveChannel, incomingMqttMsg) + else + println("undefined condition ", @__FILE__, " ", @__LINE__) + end +end + +mqttInstance = GeneralUtils.mqttClientInstance_v2( + config[:mqttServerInfo][:broker], + config[:servicetopic][:mqtttopic], + mqttMsgReceiveChannel, + keepaliveChannel, + onMsgCallback_1 +) + +println("ready!") + +# ------------------------------------------------------------------------------------------------ # +# this service main loop # +# ------------------------------------------------------------------------------------------------ # +function main() + sessiontimeout = 1*1*10 # timeout in minutes + checkSessionTimeout = 10 # minutes + clearedSessionTimestamp = Dates.now() + lastMsgId = nothing + while true + + # check if mqtt connection is still up + _ = GeneralUtils.checkMqttConnection!(mqttInstance; keepaliveCheckInterval=30) + + # check for new session + if isready(mqttMsgReceiveChannel[:ch1]) + msg = popfirst!(mqttMsgReceiveChannel[:ch1]) + # println("~~~ new msg ", msg[:payload]) + + # @spawn new runAgentInstance and store it in sessionDict + # use agent's frontend id because 1 backend agent per 1 frontend session + sessionId = msg[:msgMeta][:senderId] + sessionId = replace(sessionId, "-" => "_") # julia can't use "-" in a dict key + msgId = msg[:msgMeta][:msgId] + + if msgId != lastMsgId && sessionId ∉ keys(sessionDict) + lastMsgId = msgId + inputch = Channel{Dict}(8) + outputch = Channel{Dict}(8) + process = @spawn runServiceInstance(inputch, outputch, config, sessiontimeout) + # process = runServiceInstance(inputch, outputch, config, sessiontimeout) #XXX use spawn version + println("~~ instantiate session success") + + # call runAgentInstance() and store it in sessionDict to be able to check on it later + sessionDict[sessionId] = Dict( + :inputchannel=> inputch, + :outputchannel=> outputch, + :process=> process, + ) + + put!(sessionDict[sessionId][:inputchannel], msg) + else + + put!(sessionDict[sessionId][:inputchannel], msg) + end + end + + # check for process completed msg in serviceInternalTopic and delete it from sessionDict + # self terminate if too long inactivity + timediff = GeneralUtils.timedifference(clearedSessionTimestamp, Dates.now(), "minutes") + if timediff > checkSessionTimeout + for (sessionId, v) in sessionDict + if isready(v[:outputchannel]) + result = take!(v[:outputchannel]) + if result[:exitreason] == "timeout" + println("sessionId $(sessionId) has been deleted because it is timed out") + delete!(sessionDict, sessionId) + end + end + end + clearedSessionTimestamp = Dates.now() + end + + # sleep is needed because MQTTClient use async. while true loop leave no + # chance for control to switch to on_msg() + sleep(1) + end +end + +main() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/main.jl b/app/main.jl new file mode 100644 index 0000000..0fb069f --- /dev/null +++ b/app/main.jl @@ -0,0 +1,749 @@ + +using JSON, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames, NATS, GeneralUtils +using Base.Threads + +# ---------------------------------------------- 100 --------------------------------------------- # + + +""" Expected incomming MQTT message format for this service: + { + "msgMeta": { + "msgPurpose": "updateStatus", + "requestresponse": "request", + "timestamp": "2024-03-29T05:8:48.362", + "replyToMsgId": null, + "receiverId": null, + "getpost": "get", + "msgId": "e5c09bd8-7100-4e4e-bb43-05bee589a22c", + "acknowledgestatus": null, + "sendTopic": "/agent/wine/backend/chat/api/v1/prompt", + "receiverName": "agent-wine-backend", + "replyTopic": "/agent/wine/frontend/chat/api/v1/txt/receive", + "senderName": "agent-wine-frontend-chat", + "senderId": "0938a757-e0ee-40a9-8355-5e24906a87cd" + }, + "payload" : { + "text": "hello" + } + + } +""" + + + + + +# load config +config = copy(JSON.parsefile("./config.json")) + +function executeSQL(sql::T) where {T<:AbstractString} + DBconnection = LibPQ.Connection("host=$(config[:db][:host]) port=$(config[:db][:port]) dbname=$(config[:db][:dbname]) user=$(config[:db][:user]) password=$(config[:db][:password])") + result = LibPQ.execute(DBconnection, sql) + LibPQ.close(DBconnection) + return result +end + +function listAllTableColumns(tablename::String)::Vector + sql = + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = '$tablename'; + """ + + response = executeSQL(sql) + df = DataFrame(response) + return Symbol.(df[:, 1]) +end + + +function load_winetable(args::Dict) + tablename = "wine" + sql = + """ + SELECT * + FROM $tablename; + """ + + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + return df + else + return nothing + end +end + +function insert_masterWineDB(args::Dict) + tablename = "wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateInsertSQL(tablename, columnToUpdate, args) + println("insert_masterWineDB() SQL: $sql") + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function delete_masterWineDB(args::Dict) + tablename = "wine" + + sql = + """ + DELETE FROM $tablename + WHERE wine_id = '$(args[:wine_id])'; + """ + println("delete_masterWineDB() SQL: $sql") + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function edit_masterWineDB(args::Dict) + tablename = "wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateUpdateSQL(tablename, columnToUpdate, args, [:wine_id]) + println("") + println("edit_masterWineDB() SQL: $sql") + + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function search_masterWineTable(args::Dict) + tablename = "wine" + _searchkeyword = args[:searchkeyword] + searchkeyword1 = split(_searchkeyword, "'") # postgres not support search keyword containing ' + searchkeyword_length = length.(searchkeyword1) + _, searchkeywordIndex, _ = GeneralUtils.findMax(searchkeyword_length) + searchkeyword = searchkeyword1[searchkeywordIndex] + + columnname = args[:searchcolumn] + + # check parameters + if searchkeyword == "" + println("Error, search keyword is empty") + return "Error, search keyword is empty" + elseif columnname == "" + println("Error, search column name is empty") + return "Error, search column name is empty" + elseif isa(columnname, Number) + println("Error, search search column name must be string") + return "Error, search search column name must be string" + end + + sql = + if searchkeyword == "*" + """ + SELECT * + FROM $tablename; + """ + else + """ + SELECT * + FROM $tablename + WHERE $columnname ILIKE '%$searchkeyword%' LIMIT 1000; + """ + end + println("~~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function load_retailerWineInventory(args::Dict) + retailer_id = args[:retailerid] + if length(retailer_id) == 0 + println("No retailer_id provided ", @__FILE__, " ", @__LINE__) + return nothing + end + sql = + """ + SELECT + w.wine_id, + w.winery, + w.wine_name, + w.vintage, + w.grape, + w.wine_type, + w.region, + w.country, + w.created_time, + w.updated_time, + rw.price, + rw.currency + FROM + wine w + JOIN + retailer_wine rw ON w.wine_id = rw.wine_id + WHERE + rw.retailer_id = '$retailer_id'; + """ + # sql = + # """ + # SELECT + # w.wine_id, + # w.wine_name, + # w.winery, + # w.region, + # w.country, + # w.wine_type, + # w.grape, + # w.serving_temperature, + # w.intensity, + # w.sweetness, + # w.tannin, + # w.acidity, + # w.fizziness, + # w.tasting_notes, + # w.note, + # w.other_attributes, + # w.created_time, + # w.updated_time, + # w.description, + # rw.vintage, + # rw.price, + # rw.currency + # FROM + # wine w + # JOIN + # retailer_wine rw ON w.wine_id = rw.wine_id + # WHERE + # rw.retailer_id = '$retailer_id'; + # """ + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + return df + else + return nothing + end +end + +function insert_retailerWineInventory(args::Dict) + tablename = "retailer_wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateInsertSQL(tablename, columnToUpdate, args) + + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + +function delete_retailerWineInventory(args::Dict) + tablename = "retailer_wine" + + sql = + """ + DELETE FROM $tablename + WHERE retailer_id = '$(args[:retailer_id])' AND wine_id = '$(args[:wine_id])'; + """ + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + +function edit_retailerWineInventory(args::Dict) + # result = delete_retailerWineInventory(args) + # result = insert_retailerWineInventory(args) + + tablename = "retailer_wine" + columnToUpdate = listAllTableColumns(tablename) + sql = GeneralUtils.generateUpdateSQL(tablename, columnToUpdate, args, [:retailer_id, :wine_id]) + println("~~ sql ", sql) + response = executeSQL(sql) + df = DataFrame(response) + row, col = size(df) + if row != 0 + vd = GeneralUtils.dfToVectorDict(df) + return vd + else + return nothing + end +end + + + + +# ---------------------------------------------- 100 --------------------------------------------- # +function runServiceInstance( + receiveUserMsgChannel::Channel, + outputchannel::Channel, + config::Dict, + timeout::Int64, + ) + workDict = Dict() + latestUserMsgTimeStamp::DateTime = Dates.now() + + while true + # check for new user message + if isready(receiveUserMsgChannel) + incomingMsg = take!(receiveUserMsgChannel) + incomingPayload = incomingMsg[:payload] + latestUserMsgTimeStamp = Dates.now() + println("") + println("<-- incomingMsg ", @__FILE__, " ", @__LINE__) + println(incomingMsg) + + # sending msg back to sender i.e. LINE + msgMeta = GeneralUtils.generate_msgMeta( + incomingMsg[:msgMeta][:replyTopic]; + senderName = "wine_assistant_backend_db", + senderId= GeneralUtils.uuid4snakecase(), + replyToMsgId= incomingMsg[:msgMeta][:msgId], + mqttBrokerAddress= config[:mqttServerInfo][:broker], + mqttBrokerPort= config[:mqttServerInfo][:port], + ) + + #[WORKING] add other DB call function here + if incomingPayload[:functioncall] == "search_masterWineTable" + result = GeneralUtils.timeout(search_masterWineTable, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + # println("") + # println("-~~~ outgoingMsg ", @__FILE__, " ", @__LINE__) + # pprint(outgoingMsg) + + elseif incomingPayload[:functioncall] == "load_retailerWineInventory" + println("load_retailerWineInventory()") + # incomingMsg is requesting metadata by not having :dataTransferSessionID + if !haskey(incomingPayload, :dataTransferSessionID) + + # load data + df = load_retailerWineInventory(incomingPayload[:args]) + vd = GeneralUtils.dfToVectorDict(df) + disvd = GeneralUtils.disintegrate_vectorDict(vd, 100) + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg; data=disvd) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + + # merge dictionary so that keys are on the same level + for (k, v) in result + outgoingMsg[:payload][k] = v + end + + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + else + + # call dataTransferOverMQTT_sender + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + for (k, v) in result + outgoingMsg[:payload][k] = v + end + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + end + + # elseif incomingPayload[:functioncall] == "load_retailerWineInventory" + # result = GeneralUtils.timeout(load_retailerWineInventory, 30; + # fargs=incomingPayload[:args]) + + # outgoingMsg = Dict( + # :msgMeta=> msgMeta, + # :payload=> Dict( + # :functioncall=> incomingPayload[:functioncall], + # :result=> result + # ) + # ) + # _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "insert_retailerWineInventory" + println("insert_retailerWineInventory()") + result = GeneralUtils.timeout(insert_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "delete_retailerWineInventory" + println("delete_retailerWineInventory()") + retailer_id = incomingPayload[:args][:retailer_id] + wine_id = incomingPayload[:args][:wine_id] + if length(retailer_id) != 0 && length(wine_id) != 0 + result = GeneralUtils.timeout(delete_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + else + println("Skipped call, insufficient args for delete_retailerWineInventory() retailer_id: $retailer_id wine_id: $wine_id") + end + + elseif incomingPayload[:functioncall] == "edit_retailerWineInventory" + println("edit_retailerWineInventory()") + retailer_id = incomingPayload[:args][:retailer_id] + wine_id = incomingPayload[:args][:wine_id] + if length(retailer_id) != 0 && length(wine_id) != 0 + result = GeneralUtils.timeout(edit_retailerWineInventory, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + else + println("Skipped call, insufficient args for edit_retailerWineInventory() retailer_id: $retailer_id wine_id: $wine_id") + end + + elseif incomingPayload[:functioncall] == "load_winetable" + println("load_winetable()") + # incomingMsg is requesting metadata by not having :dataTransferSessionID + if !haskey(incomingPayload, :dataTransferSessionID) + # load data + df = load_winetable(incomingPayload[:args]) + vd = GeneralUtils.dfToVectorDict(df) + println(typeof(vd)) + disvd = GeneralUtils.disintegrate_vectorDict(vd, 100) + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg; data=disvd) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + + for (k, v) in result + outgoingMsg[:payload][k] = v + end + + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + pprintln(outgoingMsg) + + else + + # call dataTransferOverMQTT_sender + result = GeneralUtils.dataTransferOverMQTT_sender(workDict, incomingMsg) + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict{Symbol, Any}( + :functioncall=> incomingPayload[:functioncall], + ) + ) + for (k, v) in result + outgoingMsg[:payload][k] = v + end + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + end + + elseif incomingPayload[:functioncall] == "insert_masterWineDB" + println("insert_masterWineDB()") + result = GeneralUtils.timeout(insert_masterWineDB, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "delete_masterWineDB" + result = GeneralUtils.timeout(delete_masterWineDB, 30; + fargs=incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + elseif incomingPayload[:functioncall] == "edit_masterWineDB" + println("edit_masterWineDB()") + result = GeneralUtils.timeout(edit_masterWineDB, 30; + fargs=incomingPayload[:args]) + # result = edit_masterWineDB(incomingPayload[:args]) + + outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> incomingPayload[:functioncall], + :result=> result + ) + ) + _ = GeneralUtils.sendMqttMsg(outgoingMsg) + + else + println("") + println("~~~ The requested function is not defined ", @__FILE__, " ", @__LINE__) + end + end + + # self terminate if too long inactivity + timediff = GeneralUtils.timedifference(latestUserMsgTimeStamp, Dates.now(), "minutes") + if timediff > timeout + result = Dict(:exitreason=>"timeout", :timestamp=>Dates.now()) + put!(outputchannel, result) + disconnect(a.mqttClient) + break + else + sleep(1) # allowing on_msg_2, asyncmove above and other process to run + end + end +end + +sessionDict = Dict{String,Any}() +mqttMsgReceiveChannel = (ch1=Channel(8),) # store msg that coming into servicetopic +keepaliveChannel::Channel{Dict} = Channel{Dict}(8) + +# Define the callback for receiving messages. +function onMsgCallback_1(msg) + incomingNatsMsg = JSON.parse(String(msg.data)) + + if occursin("db", msg.subject) + put!(mqttMsgReceiveChannel[:ch1], incomingNatsMsg) + elseif occursin("keepalive", msg.subject) + put!(keepaliveChannel, incomingNatsMsg) + else + println("undefined condition ", @__FILE__, " ", @__LINE__) + end +end + +# Initialize NATS client +import NATS +nc = NATS.Client() + +# Connect to NATS server +url = "nats://$(config[:mqttServerInfo][:broker]):$(config[:mqttServerInfo][:port])" +@async begin + await(nc.connect(url)) + # Subscribe to the MQTT topic (compatibility) + await(nc.subscribe(config[:servicetopic][:mqtttopic], onMsgCallback_1)) +end + +# Keep the channels for compatibility +mqttMsgReceiveChannel = (ch1=Channel(8),) +keepaliveChannel::Channel{Dict} = Channel{Dict}(8) + +println("ready!") + +# ------------------------------------------------------------------------------------------------ # +# this service main loop # +# ------------------------------------------------------------------------------------------------ # +function main() + sessiontimeout = 1*1*10 # timeout in minutes + checkSessionTimeout = 10 # minutes + clearedSessionTimestamp = Dates.now() + lastMsgId = nothing + while true + + # check if mqtt connection is still up + _ = GeneralUtils.checkMqttConnection!(mqttInstance; keepaliveCheckInterval=30) + + # check for new session + if isready(mqttMsgReceiveChannel[:ch1]) + msg = popfirst!(mqttMsgReceiveChannel[:ch1]) + # println("~~~ new msg ", msg[:payload]) + + # @spawn new runAgentInstance and store it in sessionDict + # use agent's frontend id because 1 backend agent per 1 frontend session + sessionId = msg[:msgMeta][:senderId] + sessionId = replace(sessionId, "-" => "_") # julia can't use "-" in a dict key + msgId = msg[:msgMeta][:msgId] + + if msgId != lastMsgId && sessionId ∉ keys(sessionDict) + lastMsgId = msgId + inputch = Channel{Dict}(8) + outputch = Channel{Dict}(8) + process = @spawn runServiceInstance(inputch, outputch, config, sessiontimeout) + # process = runServiceInstance(inputch, outputch, config, sessiontimeout) #XXX use spawn version + println("~~ instantiate session success") + + # call runAgentInstance() and store it in sessionDict to be able to check on it later + sessionDict[sessionId] = Dict( + :inputchannel=> inputch, + :outputchannel=> outputch, + :process=> process, + ) + + put!(sessionDict[sessionId][:inputchannel], msg) + else + + put!(sessionDict[sessionId][:inputchannel], msg) + end + end + + # check for process completed msg in serviceInternalTopic and delete it from sessionDict + # self terminate if too long inactivity + timediff = GeneralUtils.timedifference(clearedSessionTimestamp, Dates.now(), "minutes") + if timediff > checkSessionTimeout + for (sessionId, v) in sessionDict + if isready(v[:outputchannel]) + result = take!(v[:outputchannel]) + if result[:exitreason] == "timeout" + println("sessionId $(sessionId) has been deleted because it is timed out") + delete!(sessionDict, sessionId) + end + end + end + clearedSessionTimestamp = Dates.now() + end + + # sleep is needed because MQTTClient use async. while true loop leave no + # chance for control to switch to on_msg() + sleep(1) + end +end + +main() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/required_python_packages.txt b/app/required_python_packages.txt new file mode 100644 index 0000000..39d3dfa --- /dev/null +++ b/app/required_python_packages.txt @@ -0,0 +1,2 @@ +# Base ---------------------------------------- +python-dev-tools \ No newline at end of file diff --git a/app/spawn_example.jl b/app/spawn_example.jl new file mode 100644 index 0000000..4f9b354 --- /dev/null +++ b/app/spawn_example.jl @@ -0,0 +1,94 @@ +using Revise # remove when this package is completed +using MQTTClient, GeneralUtils +using Base.Threads + +# --------------------------------------- code width = 100 --------------------------------------- # + + + + + + + +function count_and_listen() + total_count = 0 + + # MQTT on-message function + function on_message(client::MQTTClient.MQTTClient, message::MQTTClient.MQTTMessage) + msg = String(message.payload) + println("Received MQTT message: $msg") + try + num = parse(Int, msg) + total_count += num + catch e + println("Error parsing message: $e") + end + end + + # Connect to MQTT broker + client = MQTTClient.MQTTClient("client-1", "tcp://localhost:1883") + connect(client) + subscribe(client, "topic", on_message) + + # Count from 1 to 1,000,000,000 + for i in 1:1_000_000_000 + total_count += i + end + + # Disconnect from MQTT broker + unsubscribe(client, "topic") + disconnect(client) + + return total_count +end + +# Spawn a process to run count_and_listen function +task = @spawn count_and_listen() + + + +# Get the result +result = fetch(task) + +# Stop the spawned task +interrupt(task) + + + + + + + + + + + + + +using MQTTClient +broker = "test.mosquitto.org" + +#Define the callback for receiving messages. +function on_msg(topic, payload) + info("Received message topic: [", topic, "] payload: [", String(payload), "]") +end + +#Instantiate a client and connection. +client, connection = MakeConnection(broker, 1883) +connect(client, connection) +#Set retain to true so we can receive a message from the broker once we subscribe +#to this topic. +publish(client, "jlExample", "Hello World!", retain=true) +#Subscribe to the topic we sent a retained message to. +subscribe(client, "jlExample", on_msg, qos=QOS_1) +#Unsubscribe from the topic +unsubscribe(client, "jlExample") +#Disconnect from the broker. Not strictly needed as the broker will also +#disconnect us if the socket is closed. But this is considered good form +#and needed if you want to resume this session later. +disconnect(client) + + + + + diff --git a/app/testapp/runtests.jl b/app/testapp/runtests.jl new file mode 100644 index 0000000..0fae5a4 --- /dev/null +++ b/app/testapp/runtests.jl @@ -0,0 +1,140 @@ +# ---------------------------------------------- 100 --------------------------------------------- # +using JSON3, MQTTClient, Dates, UUIDs, PrettyPrinting, LibPQ, Base64, DataFrames +using GeneralUtils + +config = copy(JSON3.read("config.json")) + +msgMeta = GeneralUtils.generate_msgMeta( + "/yiem_branch_1/agent/wine/backend/db/api/v1/testing"; + senderName = "wine_assistant_admin", + senderId= string(uuid4()), + mqttBrokerAddress= config[:mqttServerInfo][:broker], + mqttBrokerPort= config[:mqttServerInfo][:port], +) + +outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :functioncall=> "search_materWineTable", + :args=> Dict( + :columnname=> "wine_name", + :searchkeyword=> "Yarra", + ) + + ) +) + +raw = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) + + + + + + + + + + + + + + + + +sql = +""" +SELECT * FROM wine WHERE wine_name ILIKE '%yarra%'; +""" +DBconnection = LibPQ.Connection("host=192.168.88.12 port=5432 dbname=yiem_wine_assistant user=yiem password=yiem@Postgres_0.0") +result = LibPQ.execute(DBconnection, sql) +LibPQ.close(DBconnection) + +a = columntable(result) + + + +""" Convert a DataFrame into a list of JSON rows. + +# Arguments + - `df::DataFrame` + The input DataFrame to be converted. + +# Return + - `rows::Vector{Dict{String, Any}}` + A vector of dictionaries, where each dictionary represents a row in JSON format. + +# Example + ```jldoctest + julia> using DataFrame, JSON3 + julia> df = DataFrame(A = [1, 2, 3], B = ["apple", "banana", "cherry"]) + julia> json_rows = dfToJSONRows(df) + ``` + +# Signature +""" +function dfToJSONRows(df::DataFrame) + rows = [] + for row in eachrow(df) + json_row = Dict{String, Any}() + for col in names(df) + json_row[col] = row[col] + end + push!(rows, json_row) + end + return rows +end + + +open("d.json", "w") do io + JSON3.pretty(io, result) +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/testapp/testing.jl b/app/testapp/testing.jl new file mode 100644 index 0000000..59ff4c8 --- /dev/null +++ b/app/testapp/testing.jl @@ -0,0 +1,80 @@ +using Revise # remove when this package is completed +using YiemAgent, GeneralUtils, JSON3, MQTTClient, Dates, UUIDs +using Base.Threads + +# ---------------------------------------------- 100 --------------------------------------------- # + +config = copy(JSON3.read("config.json")) + +instanceInternalTopic = config[:serviceInternalTopic][:value] * "/1" + +client, connection = MakeConnection(config[:mqttBroker][:value], 1883) + +msgMeta = GeneralUtils.generate_msgMeta( + "N/A", + replyTopic = config[:servicetopic][:value] # ask frontend reply to this instance_chat_topic + ) + +agentConfig = Dict( + :receiveprompt=>Dict( + :mqtttopic=> config[:servicetopic][:value], # topic to receive prompt i.e. frontend send msg to this topic + ), + :receiveinternal=>Dict( + :mqtttopic=> instanceInternalTopic, # receive topic for model's internal + ), + :text2text=>Dict( + :mqtttopic=> config[:text2text][:value], + ), + ) + +# Instantiate an agent +tools=Dict( # update input format + "askbox"=> Dict( + :description => "Useful for when you need to ask the user for more context. Do not ask the user their own question.", + :input => """Input is a text in JSON format.{\"Q1\": \"How are you doing?\", \"Q2\": \"How may I help you?\"}""", + :output => "" , + :func => nothing, + ), + # "winestock"=> Dict( + # :description => "A handy tool for searching wine in your inventory that match the user preferences.", + # :input => """Input is a JSON-formatted string that contains a detailed and precise search query.{\"wine type\": \"rose\", \"price\": \"max 35\", \"sweetness level\": \"sweet\", \"intensity level\": \"light bodied\", \"Tannin level\": \"low\", \"Acidity level\": \"low\"}""", + # :output => """Output are wines that match the search query in JSON format.""", + # :func => ChatAgent.winestock, + # ), + "finalanswer"=> Dict( + :description => "Useful for when you are ready to recommend wines to the user.", + :input => """{\"finalanswer\": \"some text\"}.{\"finalanswer\": \"I recommend Zena Crown Vista\"}""", + :output => "" , + :func => nothing, + ), + ) + +a = YiemAgent.sommelier( + client, + msgMeta, + agentConfig, + name= "testAgent", + id= "testid", # agent instance id + tools=tools, + ) + +response = YiemAgent.conversation(a, "hello") + + + + + + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1651190 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,71 @@ +# A Docker Compose must always start with the version tag. +# We use '3' because it's the last version. +version: "3.8" + +# You should know that Docker Compose works with services. +# 1 service = 1 container. +# For example, a service, a server, a client, a database... +# We use the keyword 'services' to start to create services. +services: + wine-db-api-v1: # service name + container_name: wine-db-api-v1 + build: ./app + image: wine-db-api-v1 + restart: unless-stopped + environment: + - TZ=Asia/Bangkok + # network_mode: "app-network" + # networks: + # - backend + ports: + - 10205:8000 + # volumes: + # - ./mountvolume:/appfolder/mountvolume + #privileged: true + deploy: + resources: + limits: + cpus: '2' + # memory: 7200M + # reservations: + # devices: + # - driver: nvidia + # device_ids: ['2', '3'] # choose between device_ids or count + # count: 1 + # capabilities: [gpu] + + # # The name of our service is "database" + # # but you can use the name of your choice. + # # Note: This may change the commands you are going to use a little bit. + # database: + # # Official Postgres image from DockerHub (we use the last version) + # image: 'postgres:latest' + + # # By default, a Postgres database is running on the 5432 port. + # # If we want to access the database from our computer (outside the container), + # # we must share the port with our computer's port. + # # The syntax is [port we want on our machine]:[port we want to retrieve in the container] + # # Note: You are free to change your computer's port, + # # but take into consideration that it will change the way + # # you are connecting to your database. + # networks: + # - backend + # ports: + # - 5432:5432 + + # environment: + # POSTGRES_USER: username # The PostgreSQL user (useful to connect to the database) + # POSTGRES_PASSWORD: password # The PostgreSQL password (useful to connect to the database) + + # volumes: + # # In this example, we share the folder `db-data` in our root repository, with the default PostgreSQL data path. + # # It means that every time the repository is modifying the data inside + # # of `/var/lib/postgresql/data/`, automatically the change will appear in `db-data`. + # # You don't need to create the `db-data` folder. Docker Compose will do it for you. + # - ./data/db-data/:/var/lib/postgresql/data/ + +# networks: +# default: + # name: https-network + # driver: bridge + # external: true