From 09615a6909789cd173eb852ea9af3b3a963aef63 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Tue, 10 Jun 2025 10:49:11 +0700 Subject: [PATCH 01/10] mark new version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index df3a90b..75bc279 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" authors = ["tonaerospace "] -version = "0.3.0" +version = "0.3.1" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" From adab61dca8595b1601639ed6b2701b8f57611c95 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Mon, 14 Jul 2025 08:54:46 +0700 Subject: [PATCH 02/10] update --- Manifest.toml | 63 +++++++++++++++++++++++++++++++++++++++++--- Project.toml | 4 +++ src/communication.jl | 6 ----- test/etc.jl | 7 ----- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index c29e4fe..15af71d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.4" +julia_version = "1.11.5" manifest_format = "2.0" -project_hash = "75c6a269a13b222c106479d2177b05facfa23f74" +project_hash = "310a130f0609e5376c1f8f5c3b09f0f87b328887" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -22,12 +22,23 @@ version = "1.11.0" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" +[[deps.BufferedStreams]] +git-tree-sha1 = "6863c5b7fc997eadcabdbaf6c5f201dc30032643" +uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" +version = "1.2.2" + [[deps.CSV]] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" version = "0.10.15" +[[deps.CodecBase]] +deps = ["TranscodingStreams"] +git-tree-sha1 = "40956acdbef3d8c7cc38cba42b56034af8f8581a" +uuid = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf" +version = "0.3.4" + [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" @@ -148,6 +159,11 @@ deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" +[[deps.HashArrayMappedTries]] +git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" +uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" +version = "0.2.0" + [[deps.HypergeometricFunctions]] deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] git-tree-sha1 = "b1c2585431c382e3fe5805874bda6aea90a95de9" @@ -279,6 +295,12 @@ deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" version = "1.11.0" +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" @@ -298,6 +320,18 @@ version = "1.11.0" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" version = "2023.12.12" +[[deps.NATS]] +deps = ["Base64", "BufferedStreams", "CodecBase", "Dates", "DocStringExtensions", "JSON3", "MbedTLS", "NanoDates", "Random", "ScopedValues", "Sockets", "Sodium", "StructTypes", "URIs"] +git-tree-sha1 = "d9d9a189fb9155a460e6b5e8966bf6a66737abf8" +uuid = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" +version = "0.1.0" + +[[deps.NanoDates]] +deps = ["Dates", "Parsers"] +git-tree-sha1 = "850a0557ae5934f6e67ac0dc5ca13d0328422d1f" +uuid = "46f1a544-deae-4307-8689-c12aa3c955c6" +version = "1.0.3" + [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" @@ -310,7 +344,7 @@ version = "0.3.27+1" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+4" +version = "0.8.5+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] @@ -423,6 +457,12 @@ version = "0.5.1+0" uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" +[[deps.ScopedValues]] +deps = ["HashArrayMappedTries", "Logging"] +git-tree-sha1 = "1147f140b4c8ddab224c94efa9569fc23d63ab44" +uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" +version = "1.3.0" + [[deps.SentinelArrays]] deps = ["Dates", "Random"] git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" @@ -437,6 +477,12 @@ version = "1.11.0" uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" +[[deps.Sodium]] +deps = ["Base64", "libsodium_jll"] +git-tree-sha1 = "907703e0d50846f300650d7225bdcab145b7bca9" +uuid = "4f5b5e99-b0ad-42cd-b47a-334e172ec8bd" +version = "1.1.2" + [[deps.SortingAlgorithms]] deps = ["DataStructures"] git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" @@ -544,6 +590,11 @@ git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.11.3" +[[deps.URIs]] +git-tree-sha1 = "24c1c558881564e2217dcf7840a8b2e10caeb0f9" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.6.0" + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -574,6 +625,12 @@ deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.11.0+0" +[[deps.libsodium_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f76d682d87eefadd3f165d8d9fda436464213142" +uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" +version = "1.0.20+3" + [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" diff --git a/Project.toml b/Project.toml index 75bc279..1e93b82 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,11 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" MQTTClient = "985f35cc-2c3d-4943-b8c1-f0931d5f0959" +NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[compat] +NATS = "0.1.0" diff --git a/src/communication.jl b/src/communication.jl index e5a3257..9aade9c 100644 --- a/src/communication.jl +++ b/src/communication.jl @@ -917,12 +917,6 @@ end - - - - - - diff --git a/test/etc.jl b/test/etc.jl index 4227812..e69de29 100644 --- a/test/etc.jl +++ b/test/etc.jl @@ -1,7 +0,0 @@ -python -> pandas -> dataframe -> csv - - - -julia -> DataFrames -> dataframe -> csv - - dict -> dataframe -> csv \ No newline at end of file From c5f3fda2ba49b79f3da6ce78b833625786446cf4 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Mon, 14 Jul 2025 13:49:04 +0700 Subject: [PATCH 03/10] update --- codesnippet/nats.jl | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 codesnippet/nats.jl diff --git a/codesnippet/nats.jl b/codesnippet/nats.jl new file mode 100644 index 0000000..26968f6 --- /dev/null +++ b/codesnippet/nats.jl @@ -0,0 +1,79 @@ + +using NATS, JSON3 +connection = NATS.connect("nats.yiem.cc:4222") +sub1 = NATS.reply(connection, "some_subject"; queue_group="group1") do msg + payload = JSON3.read(msg.payload) + println(payload) + println(msg.reply_to) + # publish(connection, msg.reply_to, "ACK") + return JSON3.write(Dict(:a=>"wassup")) + end + + + + + + + + + + +using NATS, JSON3, GeneralUtils +connection = NATS.connect("nats.yiem.cc:4222") + +msgMeta = GeneralUtils.generate_msgMeta( + "text2textinstruct_medium.inference.api.v1"; + msgPurpose= "inference", + senderName= "yiemagent", + senderId= GeneralUtils.uuid4snakecase(), + receiverName= "text2textinstruct", +) + +llmHttpTimeout = 60 +outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> Dict( + :text=> "Wassup buddy!", + :kwargs=> Dict( + :max_tokens=> 2048, + :stop=> ["<|im_end|>"], + :temperature=> 0.2, + ), + :llmHttpTimeout=>llmHttpTimeout, + ) + ) + +r = NATS.request(String, connection, "text2textinstruct_medium.inference.api.v1", + JSON3.write(outgoingMsg); timer=Timer(llmHttpTimeout)) + + + + + + + + + + + +using NATS, JSON3, GeneralUtils +connection = NATS.connect("nats.yiem.cc:4222") + +msgMeta = GeneralUtils.generate_msgMeta( + "tonpc.containerServices", + msgPurpose="reset container", + senderName= "", +) + +outgoingMsg = Dict( + :msgMeta=> msgMeta, + :payload=> "docker container restart ollama-instance-2", +) + +# may be I can't use NATS request inside NATS reply?? +_ = NATS.request(String, connection, msgMeta[:sendTopic], JSON3.write(outgoingMsg); timer=Timer(10)) + + + + + From b3e8df728714f4d0eb4ab9829634a9885b5cc7b6 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Thu, 17 Jul 2025 11:48:16 +0700 Subject: [PATCH 04/10] update --- src/interface.jl | 186 ++++++++++++++++++++++++++++++++++- src/util.jl | 250 +---------------------------------------------- 2 files changed, 189 insertions(+), 247 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 80df83e..37ff735 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -6,7 +6,7 @@ export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax, multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween, isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!, - getDictPath + getDictPath, detectKeywordVariation, textToDict using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames, CSV using ..util, ..communication @@ -1150,9 +1150,193 @@ end +""" + detectKeywordVariation(keywords::AbstractVector{String}, text::String) -> Dict{String, Union{Array, Nothing}} + +Detects and collects all case-variant occurrences of multiple keywords in the text. +This function processes each keyword individually and returns an array of matched variations for each keyword. + +# Arguments +- `keywords::AbstractVector{String}` Vector of keywords to search for +- `text::String` The text to search in + +# Returns +- `Dict{String, Array}` Returns a dictionary mapping each keyword to an array of matched variations found in the text + +# Examples + ```jldoctest + julia> detectKeywordVariation(["test", "example", "cat"], "This is a Test EXAMPLE") + Dict{String, Array}("test" => ["Test"], "example" => ["EXAMPLE"], "cat" => nothing) +""" +function detectKeywordVariation(keywords::T, text::String)::Dict{String, Union{Array, Nothing}} where {T<:AbstractVector} + kw = Dict{String, Union{Array, Nothing}}() + + # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list + for keyword in keywords + ws = detectKeywordVariation.(keyword, text) + total = sum(issomething.(ws)) + if total != 0 + kw[keyword] = ws + else + kw[keyword] = nothing + end + end + return kw +end +""" + detectKeywordVariation(keyword::String, text::String) -> Union{Nothing, Array{String}} +Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase). + +# Arguments: +- `keyword::String` The keyword to search for +- `text::String` The text to search in + +# Returns: +- `Union{Nothing, Array{String}}` Returns an array of matched keyword variations if found, otherwise returns nothing + +# Examples: + ```jldoctest + julia> detectKeywordVariation("test", "This is a Test case") + ["Test"] + + julia> detectKeywordVariation("error", "NO ERRORS FOUND") + ["ERRORS"] + + julia> detectKeywordVariation("missing", "complete data") + nothing +""" +function detectKeywordVariation(keyword::String, text::String)::Union{Nothing, Array{String}} + # Define the keyword variations to search for + wordVariations = [uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)] + # wordVariations may duplicate keyword + keyword_variations = [keyword] + for i in wordVariations + i != keyword ? push!(keyword_variations, i) : nothing + end + + _splittext = string.(strip.(split(text, " "))) + splittext = String[] + # remove . after a word + for i in _splittext + if length(i) != 0 && i[end] ∈ ['.'] + word = string(i[1:end-1]) + push!(splittext, word) + else + push!(splittext, i) + end + end + + result = String[] + for variation in keyword_variations + # if length of both word is equals then it is a whole word otherwise it is part of part of other word + r = findIndex(splittext, variation) + + if isempty(r[2]) + # skip + else + # if variation > 1 add them all so this function detect duplicate keyword + variations = [variation for i in eachindex(r[2])] + result = vcat(result, variations) + end + end + return result +end + + +""" Convert text into a dictionary with a given keywords. This function use keywords to slice + a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text. + The left most string which has no keyword will be discarded. WARNING, ordering is important + +# Arguments + - `text::String` + A text to be converted. + - `keywords::Vector{String}` + A list of keywords to be used to slice the text. + These keywords also be the resulting dict keys. +# Keyword Arguments + - `rightmarker::String` + A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", + the function will search for "plan:" otherwise the function will search for "plan". + The marker will not be in the resulting dict keys. + - `symbolkey::Bool` + If true, resulting dict's key will be Symbols, otherwise string. + - `lowercasekey::Bool` + set resulting dict's key to be lowercase + +# Return + - `d::OrderedDict` + +# Example +```jldoctest +julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep" +julia> sample_keywords = ["thought", "plan", "action"] +julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true) +julia> println(resultdict) +OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", + :plan => "wake up and going out", + :action => "1. wake up 2. eat 3. sleep") +``` + +# Signature +""" +function textToDict(text::String, detectKeywords::Vector{String}; + dictKey::Union{Vector{String}, Nothing}=nothing, + symbolkey::Bool=false, lowercasekey::Bool=false + )::OrderedDict + + # make sure this function detect variation of a work e.g. agent, Agent, AGENT + kw = [] + # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list + for keyword in detectKeywords + detected = detectKeywordVariation(keyword, text) + if detected !== nothing + push!(kw, detected) + else + error("Keyword $keyword not found in text: $text") + end + end + + od1, od2 = + if symbolkey + OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}() + else + OrderedDict{String, Any}(), OrderedDict{String, Any}() + end + + remainingtext = text + dictKey_ = reverse(dictKey) + + # process text from back to front + rkw = reverse(kw) + for (i,keyword) in enumerate(rkw) + # Find the position of the keyword in the text + keywordidx = findlast(keyword, remainingtext) + dKey = dictKey_[i] + + if keywordidx !== nothing + substr = remainingtext[keywordidx[end]+1:end] + str = string(strip(substr)) # Removes both leading and trailing whitespace. + _key = lowercasekey == true ? lowercase(dKey) : dKey + key = symbolkey == true ? Symbol(_key) : _key + od1[key] = str + remainingtext = remainingtext[1:keywordidx[1]-1] + else + error("""keyword "$keyword" not found in the provided text: $text """) + end + end + + # correct the order + ks = reverse([i for i in keys(od1)]) + for k in ks + k = symbolkey == true ? Symbol(k) : k + od2[k] = od1[k] + end + + return od2 +end diff --git a/src/util.jl b/src/util.jl index 3dabd31..16ca962 100644 --- a/src/util.jl +++ b/src/util.jl @@ -1,12 +1,12 @@ module util export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, replaceDictKeys, - findMatchingDictKey, textToDict, randstring, randstrings, timeout, + findMatchingDictKey, randstring, randstrings, timeout, dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString, - dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, + dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething, dictToString_numbering, extract_triple_backtick_text, - countGivenWords, remove_french_accents, detect_keyword, extractTextBetweenCharacter, - extractTextBetweenString, + countGivenWords, remove_french_accents, + extractTextBetweenCharacter, extractTextBetweenString, convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames @@ -244,102 +244,6 @@ function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict end -""" Convert text into a dictionary with a given keywords. This function use keywords to slice - a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text. - The left most string which has no keyword will be discarded. WARNING, ordering is important - -# Arguments - - `text::String` - A text to be converted. - - `keywords::Vector{String}` - A list of keywords to be used to slice the text. - These keywords also be the resulting dict keys. -# Keyword Arguments - - `rightmarker::String` - A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", - the function will search for "plan:" otherwise the function will search for "plan". - The marker will not be in the resulting dict keys. - - `symbolkey::Bool` - If true, resulting dict's key will be Symbols, otherwise string. - - `lowercasekey::Bool` - set resulting dict's key to be lowercase - -# Return - - `d::OrderedDict` - -# Example -```jldoctest -julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep" -julia> sample_keywords = ["thought", "plan", "action"] -julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true) -julia> println(resultdict) -OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", - :plan => "wake up and going out", - :action => "1. wake up 2. eat 3. sleep") -``` - -# Signature -""" -function textToDict(text::String, detectKeywords::Vector{String}; - dictKey::Union{Vector{String}, Nothing}=nothing, - symbolkey::Bool=false, lowercasekey::Bool=false - )::OrderedDict - - # make sure this function detect variation of a work e.g. agent, Agent, AGENT - kw = [] - # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list - for keyword in detectKeywords - detected = detect_keyword(keyword, text) - if detected !== nothing - push!(kw, detected) - else - error("Keyword $keyword not found in text.") - end - end - - od1, od2 = - if symbolkey - OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}() - else - OrderedDict{String, Any}(), OrderedDict{String, Any}() - end - - remainingtext = text - dictKey_ = reverse(dictKey) - - # process text from back to front - rkw = reverse(kw) - for (i,keyword) in enumerate(rkw) - # Find the position of the keyword in the text - keywordidx = findlast(keyword, remainingtext) - dKey = dictKey_[i] - - if keywordidx !== nothing - substr = remainingtext[keywordidx[end]+1:end] - str = string(strip(substr)) # Removes both leading and trailing whitespace. - _key = lowercasekey == true ? lowercase(dKey) : dKey - key = symbolkey == true ? Symbol(_key) : _key - od1[key] = str - remainingtext = remainingtext[1:keywordidx[1]-1] - else - error("""keyword "$keyword" not found in the provided text: $text """) - end - end - - # correct the order - ks = reverse([i for i in keys(od1)]) - for k in ks - k = symbolkey == true ? Symbol(k) : k - od2[k] = od1[k] - end - - return od2 -end - - - - - """ Generate a random string # Arguments @@ -784,152 +688,6 @@ function cuttext(range, text) end end -""" - detect_keyword(keywords::AbstractVector{String}, text::String; mode::Union{String, Nothing}=nothing, delimiter::AbstractVector=[' ', '\n', '.']) -> Dict{String, Integer} - -Detects and counts occurrences of multiple keywords in the text in different case variations (lowercase, uppercase first letter, or all uppercase). - -# Arguments -- `keywords::AbstractVector{String}` Vector of keywords to search for -- `text::String` The text to search in - -# Keyword Arguments -- `mode::Union{String, Nothing}` When set to "individual", only counts matches that are individual words (default: nothing) -- `delimiter::AbstractVector` Characters used to determine word boundaries when mode="individual" (default: [' ', '\n', '.']) - -# Returns -- `Dict{String, Integer}` Returns a dictionary mapping each keyword to its count in the text (0 if not found) - -# Examples - ```jldoctest - julia> detect_keyword(["test", "example"], "This is a Test EXAMPLE") - Dict{String, Integer}("test" => 1, "example" => 1) - - julia> detect_keyword(["cat"], "cats and category", mode="individual") - Dict{String, Integer}("cat" => 0) - - julia> detect_keyword(["error"], "No ERRORS found!") - Dict{String, Integer}("error" => 1) - ``` - -# Signature -""" -# function detect_keyword(keywords::T1, text::String; -# mode::Union{String, Nothing}=nothing, delimiter::T2=[' ', '\n', '.'] -# )::Dict{String, Integer} where {T1<:AbstractVector, T2<:AbstractVector} -# # Initialize dictionary to store keyword counts -# kwdict = Dict{String, Integer}() -# for i in keywords -# kwdict[i] = 0 -# end - -# startindex = 1 -# # Iterate through each keyword and search for matches in text -# for kw in keywords -# # Check each possible starting position in the text -# for startindex in 1:1:length(text) -# # Get the window range for current keyword at current position -# wordwindows = wordwindow(kw, startindex) -# # Extract the text slice for comparison -# cuttexts = cuttext(wordwindows, text) -# if cuttexts !== nothing -# # Try to detect keyword in current text slice -# detected_kw = detect_keyword(kw, cuttexts) -# if detected_kw !== nothing && mode === nothing -# # Increment count if keyword found and no mode restrictions -# kwdict[kw] +=1 -# elseif detected_kw !== nothing && mode === "individual" -# # For individual word mode, check word boundaries -# # Check if character before keyword is a delimiter or start of text -# checkbefore = -# if wordwindows.start > 1 && -# text[wordwindows.start-1] ∈ delimiter -# true -# elseif wordwindows.start == 1 -# true -# else -# false -# end - -# # Check if character after keyword is a delimiter or end of text -# checkafter = -# if wordwindows.stop < length(text) && -# text[wordwindows.stop+1] ∈ delimiter -# true -# elseif wordwindows.stop == length(text) -# true -# else -# false -# end -# # Only count keyword if it's a complete word -# if checkbefore && checkafter -# kwdict[kw] +=1 -# end -# end -# end -# end -# end -# return kwdict -# end - - -function detect_keyword(keywords::T, text::String)::Dict{String, Integer} where {T<:AbstractVector} - kw = Dict{String, Integer}() - splittext = string.(split(text, " ")) - # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list - for keyword in keywords - ws = detect_keyword.(keyword, splittext) - total = sum(issomething.(ws)) - if total != 0 - kw[keyword] = total - else - kw[keyword] = 0 - end - end - return kw -end - - -""" - detect_keyword(keyword::String, text::String) -> Union{Nothing, String} - -Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase). - -# Arguments: -- `keyword::String` The keyword to search for -- `text::String` The text to search in - -# Returns: -- `Union{Nothing, String}` Returns the matched keyword variation if found, otherwise returns nothing - -# Examples: - ```jldoctest - julia> detect_keyword("test", "This is a Test case") - "Test" - - julia> detect_keyword("error", "NO ERRORS FOUND") - "ERRORS" - - julia> detect_keyword("missing", "complete data") - nothing - ``` - -# Signature -""" -function detect_keyword(keyword::String, text::String)::Union{Nothing, String} - # Define the keyword variations to search for - keyword_variations = [keyword, uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)] - - # Check if any of the keyword variations are in the text - for variation in keyword_variations - if occursin(variation, text) - return variation - end - end - - # Return nothing if no variation is found - return nothing -end """ From 066d72553f6cf7147bbb1c4202386a8141f61c16 Mon Sep 17 00:00:00 2001 From: narawat lamaiin Date: Fri, 18 Jul 2025 07:54:50 +0700 Subject: [PATCH 05/10] update --- src/communication.jl | 14 +++++++------- src/interface.jl | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/communication.jl b/src/communication.jl index 9aade9c..b56609d 100644 --- a/src/communication.jl +++ b/src/communication.jl @@ -211,7 +211,7 @@ end what to do with the message. # Arguments - - `sendTopic` + - `sendTo` topic the sender sends to e.g. "/agent/wine/api/v1/prompt" # Keyword Arguments @@ -264,7 +264,7 @@ end # Signature """ function generate_msgMeta( - sendTopic::T1, # topic the sender sends to e.g. "/agent/wine/api/v1/prompt" + sendTo::T1, # topic the sender sends to e.g. "/agent/wine/api/v1/prompt" ; msgPurpose::T1= "nothing", # purpose of this message e.g. "updateStatus" senderName::T1= "nothing", # sender name (String) e.g. "agent-wine-web-frontend" @@ -292,7 +292,7 @@ function generate_msgMeta( )::Dict{Symbol, Any} where {T1<:AbstractString} msgMeta::Dict=Dict( - :sendTopic=> sendTopic, # topic the sender sends to e.g. "/agent/wine/api/v1/prompt" + :sendTo=> sendTo, # topic the sender sends to e.g. "/agent/wine/api/v1/prompt" :msgPurpose=> msgPurpose, # purpose of this message e.g. "updateStatus" :senderName=> senderName, # sender name (String) e.g. "agent-wine-web-frontend" :senderId=> senderId, # sender id e.g. uuid4snakecase() @@ -370,7 +370,7 @@ function isMqttConnectionAlive(mqttInstance::T)::Bool where {T<:mqttClientInstan ) ) - publish(mqttInstance.client, keepaliveMsg[:msgMeta][:sendTopic], + publish(mqttInstance.client, keepaliveMsg[:msgMeta][:sendTo], JSON3.write(keepaliveMsg)) timediff = 0 starttime = Dates.now() @@ -524,7 +524,7 @@ end function sendMqttMsg(mqttInstance::mqttClientInstance, outgoingMsg::Dict{Symbol, T} )::NamedTuple where {T<:Any} try - publish(mqttInstance.client, outgoingMsg[:msgMeta][:sendTopic], JSON3.write(outgoingMsg)) + publish(mqttInstance.client, outgoingMsg[:msgMeta][:sendTo], JSON3.write(outgoingMsg)) return (success=true, error=nothing) catch e return (success=false, error=e) @@ -740,7 +740,7 @@ function dataTransferOverMQTT_sender(workDict::Dict, incomingMsg::Dict; data=not :timestamp => "Tue Oct 01 2024 16:22:12 GMT+0700 (เวลาอินโดจีน)", :replyToMsgId => nothing, :acknowledgestatus => nothing, - :sendTopic => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", + :sendTo => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", :mqttBrokerPort => nothing, :receiverName => "agent_wine_backend", :replyTopic => "/agent/wine/frontend/514b634a_90b9_4e07_891d_fdf55b1aed25", @@ -807,7 +807,7 @@ function dataTransferOverMQTT_sender(workDict::Dict, incomingMsg::Dict; data=not :timestamp => "Tue Oct 01 2024 16:34:06 GMT+0700 (เวลาอินโดจีน)", :replyToMsgId => nothing, :acknowledgestatus => nothing, - :sendTopic => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", + :sendTo => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", :mqttBrokerPort => nothing, :receiverName => "agent_wine_backend", :replyTopic => "/agent/wine/frontend/514b634a_90b9_4e07_891d_fdf55b1aed25", diff --git a/src/interface.jl b/src/interface.jl index 37ff735..f7c5672 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -1298,6 +1298,9 @@ function textToDict(text::String, detectKeywords::Vector{String}; error("Keyword $keyword not found in text: $text") end end + if typeof(kw[1]) <: AbstractArray + kw = reduce(vcat, kw) + end od1, od2 = if symbolkey From 13fcf06503f93b2619ebaf56f063bae293c55325 Mon Sep 17 00:00:00 2001 From: tonaerospace Date: Wed, 23 Jul 2025 07:10:28 +0700 Subject: [PATCH 06/10] update --- codesnippet/nats.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codesnippet/nats.jl b/codesnippet/nats.jl index 26968f6..1df3438 100644 --- a/codesnippet/nats.jl +++ b/codesnippet/nats.jl @@ -2,7 +2,7 @@ using NATS, JSON3 connection = NATS.connect("nats.yiem.cc:4222") sub1 = NATS.reply(connection, "some_subject"; queue_group="group1") do msg - payload = JSON3.read(msg.payload) + payload = copy(JSON3.read(msg.payload)) println(payload) println(msg.reply_to) # publish(connection, msg.reply_to, "ACK") @@ -71,7 +71,7 @@ outgoingMsg = Dict( ) # may be I can't use NATS request inside NATS reply?? -_ = NATS.request(String, connection, msgMeta[:sendTopic], JSON3.write(outgoingMsg); timer=Timer(10)) +r = NATS.request(String, connection, msgMeta[:sendTopic], JSON3.write(outgoingMsg); timer=Timer(10)) From 0e36b8db90857113608f6ff6421cd1a9fed4ba2b Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 1 Aug 2025 06:04:17 +0700 Subject: [PATCH 07/10] remove MQTT dependency --- Manifest.toml | 17 +- Project.toml | 1 - src/GeneralUtils.jl | 2 +- src/communication.jl | 1311 +++++++++++++++++++++--------------------- src/dbUtil.jl | 2 +- src/interface.jl | 2 +- src/util.jl | 2 +- 7 files changed, 661 insertions(+), 676 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 15af71d..e55c21a 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.5" manifest_format = "2.0" -project_hash = "310a130f0609e5376c1f8f5c3b09f0f87b328887" +project_hash = "a942446c2f26ef72d0c4b0ca522e0adcf709ce4e" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -92,11 +92,6 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" -[[deps.Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" - [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] git-tree-sha1 = "3101c32aab536e7a27b1763c0797dba151b899ad" @@ -280,16 +275,6 @@ version = "0.3.28" uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" -[[deps.MQTTClient]] -deps = ["Distributed", "Random", "Sockets"] -git-tree-sha1 = "f2597b290d4bf17b577346153cd2ddf9accb5c26" -uuid = "985f35cc-2c3d-4943-b8c1-f0931d5f0959" -version = "0.3.1" -weakdeps = ["PrecompileTools"] - - [deps.MQTTClient.extensions] - PrecompileMQTT = "PrecompileTools" - [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" diff --git a/Project.toml b/Project.toml index 1e93b82..5a68284 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,6 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -MQTTClient = "985f35cc-2c3d-4943-b8c1-f0931d5f0959" NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/src/GeneralUtils.jl b/src/GeneralUtils.jl index 392e4c4..41463b1 100644 --- a/src/GeneralUtils.jl +++ b/src/GeneralUtils.jl @@ -2,7 +2,7 @@ module GeneralUtils export # struct - mqttClientInstance, + # mqttClientInstance, # function noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue diff --git a/src/communication.jl b/src/communication.jl index b56609d..f44f77f 100644 --- a/src/communication.jl +++ b/src/communication.jl @@ -1,210 +1,211 @@ module communication -export generate_msgMeta, isMqttConnectionAlive, checkMqttConnection!, - sendMqttMsg, sendReceiveMqttMsg, mqttClientInstance, mqttClientInstance_v2, - dataTransferOverMQTT_sender, dataTransferOverMQTT_receiver +export generate_msgMeta + # isMqttConnectionAlive, checkMqttConnection!, + # sendMqttMsg, sendReceiveMqttMsg, mqttClientInstance, mqttClientInstance_v2, + # dataTransferOverMQTT_sender, dataTransferOverMQTT_receiver -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames, +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, SHA, PrettyPrinting using ..util # ---------------------------------------------- 100 --------------------------------------------- # -abstract type mqttClientInstance end +# abstract type mqttClientInstance end -mutable struct mqttClientInstance_v1 <: mqttClientInstance - mqttBrokerAddress::String - mqttBrokerPort::Integer - subtopic::Vector{String} - receiveDataChannel::Channel - onMsgCallback::Function - qos::MQTTClient.QOS - client::MQTTClient.Client - connection::MQTTClient.Connection - keepalivetopic::String - keepaliveChannel::Channel - keepaliveCheckInterval::Integer # second - lastTimeMqttConnCheck::Union{DateTime, Nothing} -end +# mutable struct mqttClientInstance_v1 <: mqttClientInstance +# mqttBrokerAddress::String +# mqttBrokerPort::Integer +# subtopic::Vector{String} +# receiveDataChannel::Channel +# onMsgCallback::Function +# qos::MQTTClient.QOS +# client::MQTTClient.Client +# connection::MQTTClient.Connection +# keepalivetopic::String +# keepaliveChannel::Channel +# keepaliveCheckInterval::Integer # second +# lastTimeMqttConnCheck::Union{DateTime, Nothing} +# end -function mqttClientInstance_v1( - mqttBrokerAddress::String, - mqttBrokerPort::Integer, - subtopic::Vector{String}, - receiveDataChannel::Channel, - keepaliveChannel::Channel, # user needs to specify because it has to be accessible by user-defined onMsgCallback() - onMsgCallback::Function - ; - keepalivetopic::String= "/keepalive/$(uuid4snakecase())", - keepaliveCheckInterval::Integer=30, - qos::MQTTClient.QOS=QOS_1, - ) +# function mqttClientInstance_v1( +# mqttBrokerAddress::String, +# mqttBrokerPort::Integer, +# subtopic::Vector{String}, +# receiveDataChannel::Channel, +# keepaliveChannel::Channel, # user needs to specify because it has to be accessible by user-defined onMsgCallback() +# onMsgCallback::Function +# ; +# keepalivetopic::String= "/keepalive/$(uuid4snakecase())", +# keepaliveCheckInterval::Integer=30, +# qos::MQTTClient.QOS=QOS_1, +# ) - client, connection = MQTTClient.MakeConnection(mqttBrokerAddress, mqttBrokerPort) - MQTTClient.connect(client, connection) - for i in subtopic - MQTTClient.subscribe(client, i, onMsgCallback, qos=qos) - end - MQTTClient.subscribe(client, keepalivetopic, onMsgCallback, qos=qos) +# client, connection = MQTTClient.MakeConnection(mqttBrokerAddress, mqttBrokerPort) +# MQTTClient.connect(client, connection) +# for i in subtopic +# MQTTClient.subscribe(client, i, onMsgCallback, qos=qos) +# end +# MQTTClient.subscribe(client, keepalivetopic, onMsgCallback, qos=qos) - instance = mqttClientInstance( - mqttBrokerAddress, - mqttBrokerPort, - subtopic, - receiveDataChannel, - onMsgCallback, - qos, - client, - connection, - keepalivetopic, - keepaliveChannel, - keepaliveCheckInterval, - Dates.now(), - ) +# instance = mqttClientInstance( +# mqttBrokerAddress, +# mqttBrokerPort, +# subtopic, +# receiveDataChannel, +# onMsgCallback, +# qos, +# client, +# connection, +# keepalivetopic, +# keepaliveChannel, +# keepaliveCheckInterval, +# Dates.now(), +# ) - return instance -end +# return instance +# end -mutable struct mqttClientInstance_v2 <: mqttClientInstance - mqttBrokerAddress::String - mqttBrokerPort::Integer - subtopic::Vector{String} - msgReceiveChannel - onMsgCallback::Function - qos::MQTTClient.QOS - client::MQTTClient.Client - clientId::String - connection::MQTTClient.Connection - keepalivetopic::String - keepaliveChannel::Channel - keepaliveCheckInterval::Integer # second - lastTimeMqttConnCheck::Union{DateTime, Nothing} - multiMsg::String # "single", "metadata", "datapart" - latestMsg -end -""" MQTT client v2. The difference between client v1 and v2 is v2 allows multiple - msgReceiveChannel without having to redefine mqttClientInstance again like in v1. - This is achieved by using msgReceiveChannel::NamedTuple to store multiple channel. -# Arguments - - `mqttBrokerAddress` - MQTT broker address. Can be URL or IP address - - `subtopic` - A list of topic to be subscribed Ex. ["/testopic_1", "/testopic_2"] - - `msgReceiveChannel` - A storage the mqtt client used to store incoming message. - Ex. msgReceiveChannel = (ch1=Channel(8), ch2=Channel(32)) - - `keepaliveChannel` - A channel to store keepalive message - - `onMsgCallback` - A user-defined function to handle incoming message +# mutable struct mqttClientInstance_v2 <: mqttClientInstance +# mqttBrokerAddress::String +# mqttBrokerPort::Integer +# subtopic::Vector{String} +# msgReceiveChannel +# onMsgCallback::Function +# qos::MQTTClient.QOS +# client::MQTTClient.Client +# clientId::String +# connection::MQTTClient.Connection +# keepalivetopic::String +# keepaliveChannel::Channel +# keepaliveCheckInterval::Integer # second +# lastTimeMqttConnCheck::Union{DateTime, Nothing} +# multiMsg::String # "single", "metadata", "datapart" +# latestMsg +# end +# """ MQTT client v2. The difference between client v1 and v2 is v2 allows multiple +# msgReceiveChannel without having to redefine mqttClientInstance again like in v1. +# This is achieved by using msgReceiveChannel::NamedTuple to store multiple channel. +# # Arguments +# - `mqttBrokerAddress` +# MQTT broker address. Can be URL or IP address +# - `subtopic` +# A list of topic to be subscribed Ex. ["/testopic_1", "/testopic_2"] +# - `msgReceiveChannel` +# A storage the mqtt client used to store incoming message. +# Ex. msgReceiveChannel = (ch1=Channel(8), ch2=Channel(32)) +# - `keepaliveChannel` +# A channel to store keepalive message +# - `onMsgCallback` +# A user-defined function to handle incoming message -# Keyword Arguments - - `mqttBrokerPort` - mqtt broker port - - `keepalivetopic` - A topic where keepalive message going - - `keepaliveCheckInterval` - A time interval to check whether the mqtt client still connect to mqtt broker - - `qos` - Quality of Service. Can be QOS_0, QOS_1, QOS_2 - - `clearOldMsg` - Boolean to determine if old messages should be cleared from channels +# # Keyword Arguments +# - `mqttBrokerPort` +# mqtt broker port +# - `keepalivetopic` +# A topic where keepalive message going +# - `keepaliveCheckInterval` +# A time interval to check whether the mqtt client still connect to mqtt broker +# - `qos` +# Quality of Service. Can be QOS_0, QOS_1, QOS_2 +# - `clearOldMsg` +# Boolean to determine if old messages should be cleared from channels -# Return - - `mqttInstance` - A new mqtt client +# # Return +# - `mqttInstance` +# A new mqtt client -# Example - ```jldoctest - julia> using Revise - julia> using GeneralUtils, Dates, JSON3, UUIDs - julia> mqttMsgReceiveTopic = ["/receivetopic_1", "/receivetopic_2"] - julia> mqttMsgReceiveChannel = (ch1=Channel(8), ch2=Channel(32)) # single channel Ex. (ch1=Channel(8),) - julia> keepaliveChannel = Channel(8) - julia> function onMsgCallback(topic, payload) - jobj = JSON3.read(String(payload)) - incomingMqttMsg = copy(jobj) # convert json object into julia dictionary recursively +# # Example +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, JSON3, UUIDs +# julia> mqttMsgReceiveTopic = ["/receivetopic_1", "/receivetopic_2"] +# julia> mqttMsgReceiveChannel = (ch1=Channel(8), ch2=Channel(32)) # single channel Ex. (ch1=Channel(8),) +# julia> keepaliveChannel = Channel(8) +# julia> function onMsgCallback(topic, payload) +# jobj = JSON3.read(String(payload)) +# incomingMqttMsg = copy(jobj) # convert json object into julia dictionary recursively - if occursin("topic_1", topic) - put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg) - elseif occursin("topic_2", topic) - put!(mqttMsgReceiveChannel[:ch2], incomingMqttMsg) - elseif occursin("keepalive", topic) - put!(keepaliveChannel, incomingMqttMsg) - else - println("undefined condition ", @__FILE__, " ", @__LINE__) - end - end - julia> mqttInstance = GeneralUtils.mqttClientInstance_v2( - "mqtt.yiem.cc", - mqttMsgReceiveTopic, - mqttMsgReceiveChannel, - keepaliveChannel, - onMsgCallback - ) - ``` +# if occursin("topic_1", topic) +# put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg) +# elseif occursin("topic_2", topic) +# put!(mqttMsgReceiveChannel[:ch2], incomingMqttMsg) +# elseif occursin("keepalive", topic) +# put!(keepaliveChannel, incomingMqttMsg) +# else +# println("undefined condition ", @__FILE__, " ", @__LINE__) +# end +# end +# julia> mqttInstance = GeneralUtils.mqttClientInstance_v2( +# "mqtt.yiem.cc", +# mqttMsgReceiveTopic, +# mqttMsgReceiveChannel, +# keepaliveChannel, +# onMsgCallback +# ) +# ``` -# Signature -""" -function mqttClientInstance_v2( - mqttBrokerAddress::String, - subtopic::Vector{String}, - msgReceiveChannel, # NamedTuple of channels i.e. msgReceiveChannel = (ch1=Channel(8), ch2=Channel(8)) - keepaliveChannel::Channel, # used for checkMqttConnection(). user needs to specify because it has to be accessible by user-defined onMsgCallback() - onMsgCallback::Function - ; - clientId::String="NA", - mqttBrokerPort::Integer=1883, - keepaliveTopic::String= "/keepalive/$(uuid4snakecase())", - keepaliveCheckInterval::Integer=30, - qos::MQTTClient.QOS=QOS_1, - multiMsg::String="single", #[PENDING] bad design. this info should be stored in each msgMeta - clearOldMsg::Bool=true, - ) +# # Signature +# """ +# function mqttClientInstance_v2( +# mqttBrokerAddress::String, +# subtopic::Vector{String}, +# msgReceiveChannel, # NamedTuple of channels i.e. msgReceiveChannel = (ch1=Channel(8), ch2=Channel(8)) +# keepaliveChannel::Channel, # used for checkMqttConnection(). user needs to specify because it has to be accessible by user-defined onMsgCallback() +# onMsgCallback::Function +# ; +# clientId::String="NA", +# mqttBrokerPort::Integer=1883, +# keepaliveTopic::String= "/keepalive/$(uuid4snakecase())", +# keepaliveCheckInterval::Integer=30, +# qos::MQTTClient.QOS=QOS_1, +# multiMsg::String="single", #[PENDING] bad design. this info should be stored in each msgMeta +# clearOldMsg::Bool=true, +# ) - client, connection = MQTTClient.MakeConnection(mqttBrokerAddress, mqttBrokerPort) - MQTTClient.connect(client, connection) - for i in subtopic - MQTTClient.subscribe(client, i, onMsgCallback, qos=qos) - end - MQTTClient.subscribe(client, keepaliveTopic, onMsgCallback, qos=qos) +# client, connection = MQTTClient.MakeConnection(mqttBrokerAddress, mqttBrokerPort) +# MQTTClient.connect(client, connection) +# for i in subtopic +# MQTTClient.subscribe(client, i, onMsgCallback, qos=qos) +# end +# MQTTClient.subscribe(client, keepaliveTopic, onMsgCallback, qos=qos) - keepaliveTopic = clientId == "NA" ? keepaliveTopic : "/keepalive/$clientId" +# keepaliveTopic = clientId == "NA" ? keepaliveTopic : "/keepalive/$clientId" - instance = mqttClientInstance_v2( - mqttBrokerAddress, - mqttBrokerPort, - subtopic, - msgReceiveChannel, - onMsgCallback, - qos, - client, - clientId, - connection, - keepaliveTopic, - keepaliveChannel, - keepaliveCheckInterval, - nothing, - multiMsg, - nothing, # store latest message to prevent doing the same massage twice - ) +# instance = mqttClientInstance_v2( +# mqttBrokerAddress, +# mqttBrokerPort, +# subtopic, +# msgReceiveChannel, +# onMsgCallback, +# qos, +# client, +# clientId, +# connection, +# keepaliveTopic, +# keepaliveChannel, +# keepaliveCheckInterval, +# nothing, +# multiMsg, +# nothing, # store latest message to prevent doing the same massage twice +# ) - if clearOldMsg - chnames = keys(msgReceiveChannel) - for i in chnames - totalmsg = msgReceiveChannel[i].n_avail_items #[TESTING] - if totalmsg > 0 - while isready(msgReceiveChannel[i]) - _ = take!(msgReceiveChannel[i]) - end - println("Total $totalmsg old MQTT messages cleared ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - end - end - end +# if clearOldMsg +# chnames = keys(msgReceiveChannel) +# for i in chnames +# totalmsg = msgReceiveChannel[i].n_avail_items #[TESTING] +# if totalmsg > 0 +# while isready(msgReceiveChannel[i]) +# _ = take!(msgReceiveChannel[i]) +# end +# println("Total $totalmsg old MQTT messages cleared ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# end +# end +# end - return instance -end +# return instance +# end """ Generate msgMeta to be including in a message. So the message receiver know @@ -324,551 +325,551 @@ end -""" Check mqtt server connection. +# """ Check mqtt server connection. - Arguments\n - ----- - mqttInstanceDict::Dict{Symbol, Any} - a dictionary contain mqtt instance. 1 per mqtt client. - interval::Integer - time interval to check mqtt server in seconds +# Arguments\n +# ----- +# mqttInstanceDict::Dict{Symbol, Any} +# a dictionary contain mqtt instance. 1 per mqtt client. +# interval::Integer +# time interval to check mqtt server in seconds - Return\n - ----- - isconnectionalive::Bool - true if mqtt connection is alive +# Return\n +# ----- +# isconnectionalive::Bool +# true if mqtt connection is alive - Example\n - ----- - ```jldoctest - julia> using Revise - julia> using GeneralUtils, Dates, JSON3, MQTTClient - julia> GeneralUtils.isMqttConnectionAlive(mqttInstance) - true - ``` +# Example\n +# ----- +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, JSON3, MQTTClient +# julia> GeneralUtils.isMqttConnectionAlive(mqttInstance) +# true +# ``` - Signature\n - ----- -""" -function isMqttConnectionAlive(mqttInstance::T)::Bool where {T<:mqttClientInstance} - isconnectionalive = false +# Signature\n +# ----- +# """ +# function isMqttConnectionAlive(mqttInstance::T)::Bool where {T<:mqttClientInstance} +# isconnectionalive = false - # ditch old keepalive msg is any - while isready(mqttInstance.keepaliveChannel) - _ = take!(mqttInstance.keepaliveChannel) - end +# # ditch old keepalive msg is any +# while isready(mqttInstance.keepaliveChannel) +# _ = take!(mqttInstance.keepaliveChannel) +# end - msgMeta = generate_msgMeta( - mqttInstance.keepalivetopic, - msgPurpose= "keepalive", - ) +# msgMeta = generate_msgMeta( +# mqttInstance.keepalivetopic, +# msgPurpose= "keepalive", +# ) - keepaliveMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict( - :text=>"keepalive", - ) - ) +# keepaliveMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict( +# :text=>"keepalive", +# ) +# ) - publish(mqttInstance.client, keepaliveMsg[:msgMeta][:sendTo], - JSON3.write(keepaliveMsg)) - timediff = 0 - starttime = Dates.now() - while timediff <= 5 - timediff = timedifference(starttime, Dates.now(), "seconds") - if isready(mqttInstance.keepaliveChannel) - incomingMsg = take!(mqttInstance.keepaliveChannel) - if incomingMsg[:msgMeta][:msgId] == keepaliveMsg[:msgMeta][:msgId] - # connection is alive - isconnectionalive = true - break - end - end - sleep(1) - end - return isconnectionalive -end +# publish(mqttInstance.client, keepaliveMsg[:msgMeta][:sendTo], +# JSON3.write(keepaliveMsg)) +# timediff = 0 +# starttime = Dates.now() +# while timediff <= 5 +# timediff = timedifference(starttime, Dates.now(), "seconds") +# if isready(mqttInstance.keepaliveChannel) +# incomingMsg = take!(mqttInstance.keepaliveChannel) +# if incomingMsg[:msgMeta][:msgId] == keepaliveMsg[:msgMeta][:msgId] +# # connection is alive +# isconnectionalive = true +# break +# end +# end +# sleep(1) +# end +# return isconnectionalive +# end -""" Check mqtt server connection, reconnect if disconnected. +# """ Check mqtt server connection, reconnect if disconnected. - Arguments\n - ----- - mqttInstanceDict::Dict{Symbol, Any} - a dictionary contain mqtt instance. 1 per mqtt client. - keepaliveCheckInterval::Integer - time interval to check mqtt server in seconds +# Arguments\n +# ----- +# mqttInstanceDict::Dict{Symbol, Any} +# a dictionary contain mqtt instance. 1 per mqtt client. +# keepaliveCheckInterval::Integer +# time interval to check mqtt server in seconds - Return\n - ----- - isreconnect::Bool - true if mqtt connection is reconnected +# Return\n +# ----- +# isreconnect::Bool +# true if mqtt connection is reconnected - Example\n - ----- - ```jldoctest - julia> using Revise - julia> using GeneralUtils, Dates, JSON3 - julia> GeneralUtils.checkMqttConnection!(mqttInstance, 5) - ``` +# Example\n +# ----- +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, JSON3 +# julia> GeneralUtils.checkMqttConnection!(mqttInstance, 5) +# ``` - Signature\n - ----- -""" -function checkMqttConnection!(mqttInstance::T; - keepaliveCheckInterval::Union{Integer, Nothing}=nothing)::Union{Bool, Nothing} where {T<:mqttClientInstance} +# Signature\n +# ----- +# """ +# function checkMqttConnection!(mqttInstance::T; +# keepaliveCheckInterval::Union{Integer, Nothing}=nothing)::Union{Bool, Nothing} where {T<:mqttClientInstance} - interval = keepaliveCheckInterval !== nothing ? keepaliveCheckInterval : mqttInstance.keepaliveCheckInterval +# interval = keepaliveCheckInterval !== nothing ? keepaliveCheckInterval : mqttInstance.keepaliveCheckInterval - # check if mqtt connection is still up - intervaldiff = - if mqttInstance.lastTimeMqttConnCheck !== nothing - timedifference(mqttInstance.lastTimeMqttConnCheck, Dates.now(), "seconds") - else - Inf - end +# # check if mqtt connection is still up +# intervaldiff = +# if mqttInstance.lastTimeMqttConnCheck !== nothing +# timedifference(mqttInstance.lastTimeMqttConnCheck, Dates.now(), "seconds") +# else +# Inf +# end - if intervaldiff > interval - connectionStatusStart = isMqttConnectionAlive(mqttInstance) # a flag to note whether the connection status has changed from false to true - while true - mqttConnStatus = isMqttConnectionAlive(mqttInstance) - if mqttConnStatus == false - sleep(5) # wait - println("MQTT connection disconnected, attemping to reconnect $(Dates.now()) at $(mqttInstance.mqttBrokerAddress):$(mqttInstance.mqttBrokerPort)") - # use new client to reconnect instead of the previous one because I don't want to modify MQTTClient.jl yet - mqttInstance.client, mqttInstance.connection = - MakeConnection(mqttInstance.mqttBrokerAddress, - mqttInstance.mqttBrokerPort) - try - connect(mqttInstance.client, mqttInstance.connection) - for topic in mqttInstance.subtopic - subscribe(mqttInstance.client, topic, mqttInstance.onMsgCallback, qos=mqttInstance.qos) - end - MQTTClient.subscribe(mqttInstance.client, mqttInstance.keepalivetopic, mqttInstance.onMsgCallback, qos=mqttInstance.qos) - catch - println("Failed to reconnect MQTT broker at $(mqttInstance.mqttBrokerAddress):$(mqttInstance.mqttBrokerPort) $(Dates.now())") - end - else - mqttInstance.lastTimeMqttConnCheck = Dates.now() - if connectionStatusStart != mqttConnStatus - println("Reconnected to MQTT broker at $(mqttInstance.mqttBrokerAddress):$(mqttInstance.mqttBrokerPort) $(Dates.now())") - end - return mqttConnStatus - end - end - else - return nothing # still within check interval, no update on connection status yet - end -end +# if intervaldiff > interval +# connectionStatusStart = isMqttConnectionAlive(mqttInstance) # a flag to note whether the connection status has changed from false to true +# while true +# mqttConnStatus = isMqttConnectionAlive(mqttInstance) +# if mqttConnStatus == false +# sleep(5) # wait +# println("MQTT connection disconnected, attemping to reconnect $(Dates.now()) at $(mqttInstance.mqttBrokerAddress):$(mqttInstance.mqttBrokerPort)") +# # use new client to reconnect instead of the previous one because I don't want to modify MQTTClient.jl yet +# mqttInstance.client, mqttInstance.connection = +# MakeConnection(mqttInstance.mqttBrokerAddress, +# mqttInstance.mqttBrokerPort) +# try +# connect(mqttInstance.client, mqttInstance.connection) +# for topic in mqttInstance.subtopic +# subscribe(mqttInstance.client, topic, mqttInstance.onMsgCallback, qos=mqttInstance.qos) +# end +# MQTTClient.subscribe(mqttInstance.client, mqttInstance.keepalivetopic, mqttInstance.onMsgCallback, qos=mqttInstance.qos) +# catch +# println("Failed to reconnect MQTT broker at $(mqttInstance.mqttBrokerAddress):$(mqttInstance.mqttBrokerPort) $(Dates.now())") +# end +# else +# mqttInstance.lastTimeMqttConnCheck = Dates.now() +# if connectionStatusStart != mqttConnStatus +# println("Reconnected to MQTT broker at $(mqttInstance.mqttBrokerAddress):$(mqttInstance.mqttBrokerPort) $(Dates.now())") +# end +# return mqttConnStatus +# end +# end +# else +# return nothing # still within check interval, no update on connection status yet +# end +# end -""" Send a message to specified MQTT topic then wait for reply. +# """ Send a message to specified MQTT topic then wait for reply. -# Arguments - - `outgoingMsg::Dict` - an outgoing message +# # Arguments +# - `outgoingMsg::Dict` +# an outgoing message -# Return - - `result::NamedTuple` - ( success= true, error= nothing ) +# # Return +# - `result::NamedTuple` +# ( success= true, error= nothing ) -# Example - ```jldoctest - julia> using Revise - julia> using GeneralUtils, Dates, JSON3, UUIDs - julia> msgMeta = GeneralUtils.generate_msgMeta( - "/testtopic", - senderName= "somename", - senderId= uuid4snakecase(), - mqttBrokerAddress= "test.mosquitto.org", - mqttBrokerPort= 1883, - ) - julia> outgoingMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict("hello"=> "World"), - ) - julia> success, error = GeneralUtils.sendMqttMsg(outgoingMsg) - ``` +# # Example +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, JSON3, UUIDs +# julia> msgMeta = GeneralUtils.generate_msgMeta( +# "/testtopic", +# senderName= "somename", +# senderId= uuid4snakecase(), +# mqttBrokerAddress= "test.mosquitto.org", +# mqttBrokerPort= 1883, +# ) +# julia> outgoingMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict("hello"=> "World"), +# ) +# julia> success, error = GeneralUtils.sendMqttMsg(outgoingMsg) +# ``` -# Signature -""" -function sendMqttMsg(outgoingMsg::Dict{Symbol, T})::NamedTuple where {T<:Any} +# # Signature +# """ +# function sendMqttMsg(outgoingMsg::Dict{Symbol, T})::NamedTuple where {T<:Any} - # sendMqttMsg() doesn't need to receive msg but mqttClientInstance_v2 requires to have receive topic - mqttMsgReceiveTopic = "/GeneralUtils_sendMqttMsg/$(outgoingMsg[:msgMeta][:senderId])" +# # sendMqttMsg() doesn't need to receive msg but mqttClientInstance_v2 requires to have receive topic +# mqttMsgReceiveTopic = "/GeneralUtils_sendMqttMsg/$(outgoingMsg[:msgMeta][:senderId])" - mqttMsgReceiveChannel = (ch1=Channel(8),) - keepaliveChannel = Channel(8) +# mqttMsgReceiveChannel = (ch1=Channel(8),) +# keepaliveChannel = Channel(8) - # Define the callback for receiving messages. - function onMsgCallback(topic, payload) - jobj = JSON3.read(String(payload)) - onMsg = copy(jobj) - put!(mqttMsgReceiveChannel[:ch1], onMsg) - end +# # Define the callback for receiving messages. +# function onMsgCallback(topic, payload) +# jobj = JSON3.read(String(payload)) +# onMsg = copy(jobj) +# put!(mqttMsgReceiveChannel[:ch1], onMsg) +# end - mqttInstance = mqttClientInstance_v2( - outgoingMsg[:msgMeta][:mqttBrokerAddress], - [mqttMsgReceiveTopic], - mqttMsgReceiveChannel, - keepaliveChannel, - onMsgCallback; - mqttBrokerPort=outgoingMsg[:msgMeta][:mqttBrokerPort] - ) +# mqttInstance = mqttClientInstance_v2( +# outgoingMsg[:msgMeta][:mqttBrokerAddress], +# [mqttMsgReceiveTopic], +# mqttMsgReceiveChannel, +# keepaliveChannel, +# onMsgCallback; +# mqttBrokerPort=outgoingMsg[:msgMeta][:mqttBrokerPort] +# ) - response = sendMqttMsg(mqttInstance, outgoingMsg) - try disconnect(mqttInstance.client) catch end +# response = sendMqttMsg(mqttInstance, outgoingMsg) +# try disconnect(mqttInstance.client) catch end - return response -end -function sendMqttMsg(mqttInstance::mqttClientInstance, outgoingMsg::Dict{Symbol, T} - )::NamedTuple where {T<:Any} - try - publish(mqttInstance.client, outgoingMsg[:msgMeta][:sendTo], JSON3.write(outgoingMsg)) - return (success=true, error=nothing) - catch e - return (success=false, error=e) - end -end +# return response +# end +# function sendMqttMsg(mqttInstance::mqttClientInstance, outgoingMsg::Dict{Symbol, T} +# )::NamedTuple where {T<:Any} +# try +# publish(mqttInstance.client, outgoingMsg[:msgMeta][:sendTo], JSON3.write(outgoingMsg)) +# return (success=true, error=nothing) +# catch e +# return (success=false, error=e) +# end +# end -""" Send a message to specified MQTT topic then wait for reply. +# """ Send a message to specified MQTT topic then wait for reply. -# Arguments - - `outgoingMsg::Dict{Symbol, T}` - an outgoing message +# # Arguments +# - `outgoingMsg::Dict{Symbol, T}` +# an outgoing message -# Keyword Arguments - - `timeout::Integer=60` - time in seconds to wait for a response before error - - `maxattempt::Integer=1` - maximum number of attempts to send and receive message +# # Keyword Arguments +# - `timeout::Integer=60` +# time in seconds to wait for a response before error +# - `maxattempt::Integer=1` +# maximum number of attempts to send and receive message -# Return - - `result::NamedTuple` - ( - success= true, # idicates whether sending MQTT message successful - error= nothing # error message - response= somemessage # response message - ) +# # Return +# - `result::NamedTuple` +# ( +# success= true, # idicates whether sending MQTT message successful +# error= nothing # error message +# response= somemessage # response message +# ) -# Example -```jldoctest -julia> using Revise -julia> using GeneralUtils, Dates, UUIDs, JSON3 -julia> msgMeta = GeneralUtils.generate_msgMeta( - "/testtopic", - senderName= "somename", - senderId= uuid4snakecase(), - mqttBrokerAddress= "mqtt.yiem.cc", - mqttBrokerPort= 1883, - ) -julia> outgoingMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict(:hello=> "World"), - ) -julia> success, error, response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) -``` +# # Example +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, UUIDs, JSON3 +# julia> msgMeta = GeneralUtils.generate_msgMeta( +# "/testtopic", +# senderName= "somename", +# senderId= uuid4snakecase(), +# mqttBrokerAddress= "mqtt.yiem.cc", +# mqttBrokerPort= 1883, +# ) +# julia> outgoingMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict(:hello=> "World"), +# ) +# julia> success, error, response = GeneralUtils.sendReceiveMqttMsg(outgoingMsg) +# ``` -# Signature -""" -function sendReceiveMqttMsg(outgoingMsg::Dict{Symbol, T}; - connectiontimeout::Integer=600, responsetimeout::Integer=60, responsemaxattempt::Integer=1)::NamedTuple where {T<:Any} - senderId = outgoingMsg[:msgMeta][:senderId] - mqttMsgReceiveTopic = "/GeneralUtils_sendReceiveMqttMsg/$senderId" - mqttMsgReceiveChannel = (ch1=Channel(8),) - keepaliveChannel = Channel(8) - outgoingMsg[:msgMeta][:replyTopic] = mqttMsgReceiveTopic +# # Signature +# """ +# function sendReceiveMqttMsg(outgoingMsg::Dict{Symbol, T}; +# connectiontimeout::Integer=600, responsetimeout::Integer=60, responsemaxattempt::Integer=1)::NamedTuple where {T<:Any} +# senderId = outgoingMsg[:msgMeta][:senderId] +# mqttMsgReceiveTopic = "/GeneralUtils_sendReceiveMqttMsg/$senderId" +# mqttMsgReceiveChannel = (ch1=Channel(8),) +# keepaliveChannel = Channel(8) +# outgoingMsg[:msgMeta][:replyTopic] = mqttMsgReceiveTopic - # Define the callback for receiving messages. - function onMsgCallback(topic, payload) - jobj = JSON3.read(String(payload)) - incomingMqttMsg = copy(jobj) +# # Define the callback for receiving messages. +# function onMsgCallback(topic, payload) +# jobj = JSON3.read(String(payload)) +# incomingMqttMsg = copy(jobj) - if occursin("GeneralUtils_sendReceiveMqttMsg", topic) - put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg) - elseif occursin("keepalive", topic) - put!(keepaliveChannel, incomingMqttMsg) - else - println("undefined condition ", @__FILE__, " ", @__LINE__) - end - end +# if occursin("GeneralUtils_sendReceiveMqttMsg", topic) +# put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg) +# elseif occursin("keepalive", topic) +# put!(keepaliveChannel, incomingMqttMsg) +# else +# println("undefined condition ", @__FILE__, " ", @__LINE__) +# end +# end - mqttInstance = nothing - attempt = 0 - starttime = Dates.now() - endtime = starttime + Second(connectiontimeout) - errormsg = nothing - while true - timenow = Dates.now() - timepass = timedifference(starttime, timenow, "seconds") - timeleft = timedifference(timenow, endtime, "seconds") +# mqttInstance = nothing +# attempt = 0 +# starttime = Dates.now() +# endtime = starttime + Second(connectiontimeout) +# errormsg = nothing +# while true +# timenow = Dates.now() +# timepass = timedifference(starttime, timenow, "seconds") +# timeleft = timedifference(timenow, endtime, "seconds") - if timepass <= connectiontimeout - attempt += 1 - attempt > 1 ? println("Attempt $attempt to connect to MQTT broker. Timed out in $timeleft seconds. $errormsg") : nothing +# if timepass <= connectiontimeout +# attempt += 1 +# attempt > 1 ? println("Attempt $attempt to connect to MQTT broker. Timed out in $timeleft seconds. $errormsg") : nothing - try - mqttInstance = mqttClientInstance_v2( - outgoingMsg[:msgMeta][:mqttBrokerAddress], - [mqttMsgReceiveTopic], - mqttMsgReceiveChannel, - keepaliveChannel, - onMsgCallback; - mqttBrokerPort=outgoingMsg[:msgMeta][:mqttBrokerPort], - clientId=senderId - ) - break - catch e - errormsg = e - sleep(5) - end - else - println("Failed to instantiate MQTT client after $timepass seconds. $errormsg") - return nothing - end - end +# try +# mqttInstance = mqttClientInstance_v2( +# outgoingMsg[:msgMeta][:mqttBrokerAddress], +# [mqttMsgReceiveTopic], +# mqttMsgReceiveChannel, +# keepaliveChannel, +# onMsgCallback; +# mqttBrokerPort=outgoingMsg[:msgMeta][:mqttBrokerPort], +# clientId=senderId +# ) +# break +# catch e +# errormsg = e +# sleep(5) +# end +# else +# println("Failed to instantiate MQTT client after $timepass seconds. $errormsg") +# return nothing +# end +# end - response = sendReceiveMqttMsg(mqttInstance, :ch1, outgoingMsg; - responsetimeout=responsetimeout, responsemaxattempt=responsemaxattempt) - try disconnect(mqttInstance.client) catch end +# response = sendReceiveMqttMsg(mqttInstance, :ch1, outgoingMsg; +# responsetimeout=responsetimeout, responsemaxattempt=responsemaxattempt) +# try disconnect(mqttInstance.client) catch end - return response -end -function sendReceiveMqttMsg(mqttInstance::mqttClientInstance_v2, receivechannel::Symbol, - outgoingMsg::Dict{Symbol, T}; responsetimeout::Integer=60, responsemaxattempt::Integer=1 - )::NamedTuple where {T<:Any} +# return response +# end +# function sendReceiveMqttMsg(mqttInstance::mqttClientInstance_v2, receivechannel::Symbol, +# outgoingMsg::Dict{Symbol, T}; responsetimeout::Integer=60, responsemaxattempt::Integer=1 +# )::NamedTuple where {T<:Any} - timepass = nothing - attempts = 1 - while attempts <= responsemaxattempt - sendMqttMsg(mqttInstance, outgoingMsg) +# timepass = nothing +# attempts = 1 +# while attempts <= responsemaxattempt +# sendMqttMsg(mqttInstance, outgoingMsg) - starttime = Dates.now() - while true - timepass = timedifference(starttime, Dates.now(), "seconds") +# starttime = Dates.now() +# while true +# timepass = timedifference(starttime, Dates.now(), "seconds") - if timepass <= responsetimeout - if isready(mqttInstance.msgReceiveChannel[receivechannel]) - incomingMsg = take!(mqttInstance.msgReceiveChannel[receivechannel]) +# if timepass <= responsetimeout +# if isready(mqttInstance.msgReceiveChannel[receivechannel]) +# incomingMsg = take!(mqttInstance.msgReceiveChannel[receivechannel]) - if incomingMsg[:msgMeta][:replyToMsgId] == outgoingMsg[:msgMeta][:msgId] - if incomingMsg[:msgMeta][:acknowledgestatus] == "NACK" - println("NACK") - break # resend the msg - else - return (success=true, error=nothing, response=incomingMsg[:payload]) - end - end - end - else - break - end - sleep(1) - end - if attempts > 1 - println("\n attempts $attempts/$responsemaxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") - pprintln(outgoingMsg) - println(" attempts $attempts/$responsemaxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") - checkMqttConnection!(mqttInstance, keepaliveCheckInterval=5) - end - attempts += 1 - end +# if incomingMsg[:msgMeta][:replyToMsgId] == outgoingMsg[:msgMeta][:msgId] +# if incomingMsg[:msgMeta][:acknowledgestatus] == "NACK" +# println("NACK") +# break # resend the msg +# else +# return (success=true, error=nothing, response=incomingMsg[:payload]) +# end +# end +# end +# else +# break +# end +# sleep(1) +# end +# if attempts > 1 +# println("\n attempts $attempts/$responsemaxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())") +# pprintln(outgoingMsg) +# println(" attempts $attempts/$responsemaxattempt ", @__FILE__, ":", @__LINE__, " $(Dates.now())\n") +# checkMqttConnection!(mqttInstance, keepaliveCheckInterval=5) +# end +# attempts += 1 +# end - return (success=false, - error="no response, timeout $timepass/$timeout, $(attempts-1) publish attempted", - response=nothing) -end +# return (success=false, +# error="no response, timeout $timepass/$timeout, $(attempts-1) publish attempted", +# response=nothing) +# end -""" Send a message to specified MQTT topic then wait for reply. +# """ Send a message to specified MQTT topic then wait for reply. -# Arguments - - `workDict::Dict` - A dictionary containing workspace data - - `incomingMsg::Dict` - The incoming message to process +# # Arguments +# - `workDict::Dict` +# A dictionary containing workspace data +# - `incomingMsg::Dict` +# The incoming message to process -# Keyword Arguments - - `data` - Optional data to be transferred. Default is nothing. +# # Keyword Arguments +# - `data` +# Optional data to be transferred. Default is nothing. -# Return - - `result::Dict` - A dictionary containing either: - - Metadata about the data transfer session if initiating a new transfer - - The requested data part if responding to a data part request +# # Return +# - `result::Dict` +# A dictionary containing either: +# - Metadata about the data transfer session if initiating a new transfer +# - The requested data part if responding to a data part request -# Example - ```jldoctest - julia> using Revise - julia> using GeneralUtils, Dates, JSON3, UUIDs - julia> msgMeta = GeneralUtils.generate_msgMeta( - "/testtopic", - senderName= "somename", - senderId= uuid4snakecase(), - mqttBrokerAddress= "test.mosquitto.org", - mqttBrokerPort= 1883, - ) - julia> outgoingMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict("hello"=> "World"), - ) - julia> success, error = GeneralUtils.sendMqttMsg(outgoingMsg) - ``` -# TODO - - [x] transfer multiple msg with the same mqtt connection. The transfer loop should be - implemented by passing mqttInstance into sendReceiveMqttMsg() so that - 1-connection is used to send all data - - [] partsize should be calculate from the fact that MQTT max size is 4MB (256MB theorectically) +# # Example +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, JSON3, UUIDs +# julia> msgMeta = GeneralUtils.generate_msgMeta( +# "/testtopic", +# senderName= "somename", +# senderId= uuid4snakecase(), +# mqttBrokerAddress= "test.mosquitto.org", +# mqttBrokerPort= 1883, +# ) +# julia> outgoingMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict("hello"=> "World"), +# ) +# julia> success, error = GeneralUtils.sendMqttMsg(outgoingMsg) +# ``` +# # TODO +# - [x] transfer multiple msg with the same mqtt connection. The transfer loop should be +# implemented by passing mqttInstance into sendReceiveMqttMsg() so that +# 1-connection is used to send all data +# - [] partsize should be calculate from the fact that MQTT max size is 4MB (256MB theorectically) -# Signature -""" -function dataTransferOverMQTT_sender(workDict::Dict, incomingMsg::Dict; data=nothing) - dataTransferSessionID = nothing +# # Signature +# """ +# function dataTransferOverMQTT_sender(workDict::Dict, incomingMsg::Dict; data=nothing) +# dataTransferSessionID = nothing - if !haskey(incomingMsg[:payload], :dataTransferSessionID) - """ Expected incomingMsg (without "dataTransferSessionID") for requesting metadata +# if !haskey(incomingMsg[:payload], :dataTransferSessionID) +# """ Expected incomingMsg (without "dataTransferSessionID") for requesting metadata - Dict(:payload => Dict(:args => Dict(), :functioncall => "load_winetable"), - :msgMeta => Dict(:msgPurpose => nothing, - :receiverId => nothing, - :getpost => nothing, - :msgId => "7fe4b765_fa2e_4c2e_9df0_9cb97e84ec32", - :requestresponse => nothing, - :msgFormatVersion => nothing, - :timestamp => "Tue Oct 01 2024 16:22:12 GMT+0700 (เวลาอินโดจีน)", - :replyToMsgId => nothing, - :acknowledgestatus => nothing, - :sendTo => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", - :mqttBrokerPort => nothing, - :receiverName => "agent_wine_backend", - :replyTopic => "/agent/wine/frontend/514b634a_90b9_4e07_891d_fdf55b1aed25", - :mqttBrokerAddress => "wss://mqtt.yiem.cc:8083", - :senderName => "agent_wine_frontend", - :senderselfnote => nothing, - :senderId => "514b634a_90b9_4e07_891d_fdf55b1aed25")) - """ +# Dict(:payload => Dict(:args => Dict(), :functioncall => "load_winetable"), +# :msgMeta => Dict(:msgPurpose => nothing, +# :receiverId => nothing, +# :getpost => nothing, +# :msgId => "7fe4b765_fa2e_4c2e_9df0_9cb97e84ec32", +# :requestresponse => nothing, +# :msgFormatVersion => nothing, +# :timestamp => "Tue Oct 01 2024 16:22:12 GMT+0700 (เวลาอินโดจีน)", +# :replyToMsgId => nothing, +# :acknowledgestatus => nothing, +# :sendTo => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", +# :mqttBrokerPort => nothing, +# :receiverName => "agent_wine_backend", +# :replyTopic => "/agent/wine/frontend/514b634a_90b9_4e07_891d_fdf55b1aed25", +# :mqttBrokerAddress => "wss://mqtt.yiem.cc:8083", +# :senderName => "agent_wine_frontend", +# :senderselfnote => nothing, +# :senderId => "514b634a_90b9_4e07_891d_fdf55b1aed25")) +# """ - if data !== nothing # if the user provide data, store it in a new session workspace - dataTransferSessionID = uuid4snakecase() - haskey(workDict, :dataTransferOverMQTT) ? nothing : workDict[:dataTransferOverMQTT] = Dict() - workDict[:dataTransferOverMQTT][dataTransferSessionID] = Dict() - s = workDict[:dataTransferOverMQTT][dataTransferSessionID] - s[:dataTransferSessionID] = dataTransferSessionID - dataDict = Dict(pairs(data)) - merge!(s, dataDict) - else - error("No data provided") - end +# if data !== nothing # if the user provide data, store it in a new session workspace +# dataTransferSessionID = uuid4snakecase() +# haskey(workDict, :dataTransferOverMQTT) ? nothing : workDict[:dataTransferOverMQTT] = Dict() +# workDict[:dataTransferOverMQTT][dataTransferSessionID] = Dict() +# s = workDict[:dataTransferOverMQTT][dataTransferSessionID] +# s[:dataTransferSessionID] = dataTransferSessionID +# dataDict = Dict(pairs(data)) +# merge!(s, dataDict) +# else +# error("No data provided") +# end - # compute hash value for each piece of data - s = workDict[:dataTransferOverMQTT][dataTransferSessionID] - s[:hashes] = [bytes2hex(sha256(JSON3.write(s[:dataparts][i]))) for i in 1:s[:totalparts]] +# # compute hash value for each piece of data +# s = workDict[:dataTransferOverMQTT][dataTransferSessionID] +# s[:hashes] = [bytes2hex(sha256(JSON3.write(s[:dataparts][i]))) for i in 1:s[:totalparts]] - # send metadata without data to receiver() - datainfo = Dict() - for (k, v) in s - if k ∉ [:dataparts] - datainfo[k] = v - end - end +# # send metadata without data to receiver() +# datainfo = Dict() +# for (k, v) in s +# if k ∉ [:dataparts] +# datainfo[k] = v +# end +# end - d = Dict( - :result=> "dataTransferOverMQTT", - :dataTransferSessionID=> dataTransferSessionID, - Symbol(dataTransferSessionID)=> datainfo - ) +# d = Dict( +# :result=> "dataTransferOverMQTT", +# :dataTransferSessionID=> dataTransferSessionID, +# Symbol(dataTransferSessionID)=> datainfo +# ) - """ Sending metadata to the requester - Dict(:result => "dataTransferOverMQTT", - :b57123aa_a4c6_4d07_ace2_4885005d51c6 => Dict(:partsize => 3, - :hashes => ["755fb25ae04890fa2a40c2b5fb5ddf05adeff6b934475998576bdf307359cd0f", - "b95ba78344ad409755a386b6ab0cb2c8ea490756ce2df401aa6e5f77cf445025", - "5b51277dd06ca255492cfc432824adf77e9e0de94179806cb130bef38e75da48"], - :datatype => "vector{Dict}", - :dataTransferSessionID => "b57123aa_a4c6_4d07_ace2_4885005d51c6", - :totalparts => 46), - :dataTransferSessionID => "b57123aa_a4c6_4d07_ace2_4885005d51c6") - """ +# """ Sending metadata to the requester +# Dict(:result => "dataTransferOverMQTT", +# :b57123aa_a4c6_4d07_ace2_4885005d51c6 => Dict(:partsize => 3, +# :hashes => ["755fb25ae04890fa2a40c2b5fb5ddf05adeff6b934475998576bdf307359cd0f", +# "b95ba78344ad409755a386b6ab0cb2c8ea490756ce2df401aa6e5f77cf445025", +# "5b51277dd06ca255492cfc432824adf77e9e0de94179806cb130bef38e75da48"], +# :datatype => "vector{Dict}", +# :dataTransferSessionID => "b57123aa_a4c6_4d07_ace2_4885005d51c6", +# :totalparts => 46), +# :dataTransferSessionID => "b57123aa_a4c6_4d07_ace2_4885005d51c6") +# """ - return d +# return d - else +# else - """ Expected incomingMsg (with "dataTransferSessionID") for requesting a data part - Dict(:payload => Dict(:dataTransferSessionID => "5543f3b9_194e_4670_b961_e51c77955e40", :args => Dict(), :requestingDataPartNumber => 1, :functioncall => "load_winetable"), - :msgMeta => Dict(:msgPurpose => nothing, - :receiverId => nothing, - :getpost => nothing, - :msgId => "fbcc3bd9_c2bc_4bde_908a_dfe88ab7a305", - :requestresponse => nothing, - :msgFormatVersion => nothing, - :timestamp => "Tue Oct 01 2024 16:34:06 GMT+0700 (เวลาอินโดจีน)", - :replyToMsgId => nothing, - :acknowledgestatus => nothing, - :sendTo => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", - :mqttBrokerPort => nothing, - :receiverName => "agent_wine_backend", - :replyTopic => "/agent/wine/frontend/514b634a_90b9_4e07_891d_fdf55b1aed25", - :mqttBrokerAddress => "wss://mqtt.yiem.cc:8083", - :senderName => "agent_wine_frontend", - :senderselfnote => nothing, - :senderId => "514b634a_90b9_4e07_891d_fdf55b1aed25")) - """ +# """ Expected incomingMsg (with "dataTransferSessionID") for requesting a data part +# Dict(:payload => Dict(:dataTransferSessionID => "5543f3b9_194e_4670_b961_e51c77955e40", :args => Dict(), :requestingDataPartNumber => 1, :functioncall => "load_winetable"), +# :msgMeta => Dict(:msgPurpose => nothing, +# :receiverId => nothing, +# :getpost => nothing, +# :msgId => "fbcc3bd9_c2bc_4bde_908a_dfe88ab7a305", +# :requestresponse => nothing, +# :msgFormatVersion => nothing, +# :timestamp => "Tue Oct 01 2024 16:34:06 GMT+0700 (เวลาอินโดจีน)", +# :replyToMsgId => nothing, +# :acknowledgestatus => nothing, +# :sendTo => "/yiem_branch_1/agent/wine/backend/db/api/v1/testing", +# :mqttBrokerPort => nothing, +# :receiverName => "agent_wine_backend", +# :replyTopic => "/agent/wine/frontend/514b634a_90b9_4e07_891d_fdf55b1aed25", +# :mqttBrokerAddress => "wss://mqtt.yiem.cc:8083", +# :senderName => "agent_wine_frontend", +# :senderselfnote => nothing, +# :senderId => "514b634a_90b9_4e07_891d_fdf55b1aed25")) +# """ - # check transfer session id - dataTransferSessionID = incomingMsg[:payload][:dataTransferSessionID] - clientRequestingDataPartNumber = - incomingMsg[:payload][:requestingDataPartNumber] - s = workDict[:dataTransferOverMQTT][dataTransferSessionID] +# # check transfer session id +# dataTransferSessionID = incomingMsg[:payload][:dataTransferSessionID] +# clientRequestingDataPartNumber = +# incomingMsg[:payload][:requestingDataPartNumber] +# s = workDict[:dataTransferOverMQTT][dataTransferSessionID] - d = Dict( - :result=> "dataTransferOverMQTT", - :dataTransferSessionID=> dataTransferSessionID, - Symbol(dataTransferSessionID)=> Dict( - :dataPartNumber=> clientRequestingDataPartNumber, - :datapart=> s[:dataparts][clientRequestingDataPartNumber] - ), - ) +# d = Dict( +# :result=> "dataTransferOverMQTT", +# :dataTransferSessionID=> dataTransferSessionID, +# Symbol(dataTransferSessionID)=> Dict( +# :dataPartNumber=> clientRequestingDataPartNumber, +# :datapart=> s[:dataparts][clientRequestingDataPartNumber] +# ), +# ) - return d - end -end +# return d +# end +# end -""" Send a message to specified MQTT topic then wait for reply. +# """ Send a message to specified MQTT topic then wait for reply. -# Arguments - - `outgoingMsg::Dict` - an outgoing message +# # Arguments +# - `outgoingMsg::Dict` +# an outgoing message -# Return - - `result::NamedTuple` - ( success= true, error= nothing ) +# # Return +# - `result::NamedTuple` +# ( success= true, error= nothing ) -# Example - ```jldoctest - julia> using Revise - julia> using GeneralUtils, Dates, JSON3, UUIDs - julia> msgMeta = GeneralUtils.generate_msgMeta( - "/testtopic", - senderName= "somename", - senderId= uuid4snakecase(), - mqttBrokerAddress= "test.mosquitto.org", - mqttBrokerPort= 1883, - ) - julia> outgoingMsg = Dict( - :msgMeta=> msgMeta, - :payload=> Dict("hello"=> "World"), - ) - julia> success, error = GeneralUtils.sendMqttMsg(outgoingMsg) - ``` -# TODO - - [PENDING] +# # Example +# ```jldoctest +# julia> using Revise +# julia> using GeneralUtils, Dates, JSON3, UUIDs +# julia> msgMeta = GeneralUtils.generate_msgMeta( +# "/testtopic", +# senderName= "somename", +# senderId= uuid4snakecase(), +# mqttBrokerAddress= "test.mosquitto.org", +# mqttBrokerPort= 1883, +# ) +# julia> outgoingMsg = Dict( +# :msgMeta=> msgMeta, +# :payload=> Dict("hello"=> "World"), +# ) +# julia> success, error = GeneralUtils.sendMqttMsg(outgoingMsg) +# ``` +# # TODO +# - [PENDING] -# Signature -""" -function dataTransferOverMQTT_receiver() +# # Signature +# """ +# function dataTransferOverMQTT_receiver() @@ -878,7 +879,7 @@ function dataTransferOverMQTT_receiver() -end +# end diff --git a/src/dbUtil.jl b/src/dbUtil.jl index 40296bd..2e4fea8 100644 --- a/src/dbUtil.jl +++ b/src/dbUtil.jl @@ -2,7 +2,7 @@ module dbUtil export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames, +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, SHA using ..util diff --git a/src/interface.jl b/src/interface.jl index f7c5672..dea5dcb 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -8,7 +8,7 @@ export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!, getDictPath, detectKeywordVariation, textToDict -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames, CSV +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV using ..util, ..communication # ---------------------------------------------- 100 --------------------------------------------- # diff --git a/src/util.jl b/src/util.jl index 16ca962..671a201 100644 --- a/src/util.jl +++ b/src/util.jl @@ -9,7 +9,7 @@ export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, rep extractTextBetweenCharacter, extractTextBetweenString, convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames +using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames # ---------------------------------------------- 100 --------------------------------------------- # From fbedd507fc8d8028e5d427c1225fa67c6da763c5 Mon Sep 17 00:00:00 2001 From: narawat Date: Sat, 22 Nov 2025 09:05:06 +0700 Subject: [PATCH 08/10] update --- Manifest.toml | 300 +++++++++++++++++++++---------------------- Project.toml | 5 +- README.md | 5 + src/GeneralUtils.jl | 9 -- src/communication.jl | 4 +- src/dbUtil.jl | 4 +- src/interface.jl | 4 +- src/llmUtil.jl | 4 +- src/util.jl | 10 +- 9 files changed, 164 insertions(+), 181 deletions(-) create mode 100644 README.md diff --git a/Manifest.toml b/Manifest.toml index e55c21a..39d3cc8 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.5" +julia_version = "1.12.1" manifest_format = "2.0" -project_hash = "a942446c2f26ef72d0c4b0ca522e0adcf709ce4e" +project_hash = "8a1366f01fdc86b8cec7588b2642a428f8711411" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -10,10 +10,6 @@ git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" version = "1.1.3" -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" - [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" @@ -41,15 +37,15 @@ version = "0.3.4" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.6" +version = "0.7.8" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.16.0" +version = "4.18.1" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] @@ -58,7 +54,7 @@ weakdeps = ["Dates", "LinearAlgebra"] [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.1.1+0" +version = "1.3.0+1" [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" @@ -72,15 +68,15 @@ version = "1.16.0" [[deps.DataFrames]] deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" +git-tree-sha1 = "d8928e9169ff76c6281f39a659f9bca3a573f24c" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.7.0" +version = "1.8.1" [[deps.DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +deps = ["OrderedCollections"] +git-tree-sha1 = "e357641bb3e0638d353c4b29ea0e40ea644066a6" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.20" +version = "0.19.3" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -94,9 +90,9 @@ version = "1.11.0" [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] -git-tree-sha1 = "3101c32aab536e7a27b1763c0797dba151b899ad" +git-tree-sha1 = "3bc002af51045ca3b47d2e1787d6ce02e68b943a" uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.113" +version = "0.25.122" [deps.Distributions.extensions] DistributionsChainRulesCoreExt = "ChainRulesCore" @@ -109,21 +105,15 @@ version = "0.25.113" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" +version = "0.9.5" [[deps.FilePathsBase]] deps = ["Compat", "Dates"] -git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361" +git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" uuid = "48062228-2e41-5def-b9a4-89aafe57970f" -version = "0.9.22" +version = "0.9.24" [deps.FilePathsBase.extensions] FilePathsBaseMmapExt = "Mmap" @@ -133,15 +123,11 @@ version = "0.9.22" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -version = "1.11.0" - [[deps.FillArrays]] deps = ["LinearAlgebra"] -git-tree-sha1 = "6a70198746448456524cb442b8af316927ff3e1a" +git-tree-sha1 = "5bfcd42851cf2f1b303f51525a54dc5e98d408a3" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.13.0" +version = "1.15.0" weakdeps = ["PDMats", "SparseArrays", "Statistics"] [deps.FillArrays.extensions] @@ -154,6 +140,12 @@ deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" +[[deps.GeneralUtils]] +deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "SHA", "UUIDs"] +path = "." +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +version = "0.3.1" + [[deps.HashArrayMappedTries]] git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" @@ -161,14 +153,14 @@ version = "0.2.0" [[deps.HypergeometricFunctions]] deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] -git-tree-sha1 = "b1c2585431c382e3fe5805874bda6aea90a95de9" +git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" -version = "0.3.25" +version = "0.3.28" [[deps.InlineStrings]] -git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" +git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.2" +version = "1.4.5" [deps.InlineStrings.extensions] ArrowTypesExt = "ArrowTypes" @@ -184,14 +176,14 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" [[deps.InvertedIndices]] -git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.3.0" +version = "1.3.1" [[deps.IrrationalConstants]] -git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" -version = "0.2.2" +version = "0.2.6" [[deps.IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" @@ -200,15 +192,27 @@ version = "1.0.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.6.1" +version = "1.7.1" + +[[deps.JSON]] +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "5b6bb73f555bc753a6153deec3717b8904f5551c" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "1.3.0" + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + + [deps.JSON.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] -git-tree-sha1 = "1d322381ef7b087548321d3f878cb4c9bd8f8f9b" +git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -version = "1.14.1" +version = "1.14.3" [deps.JSON3.extensions] JSON3ArrowExt = ["ArrowTypes"] @@ -216,36 +220,16 @@ version = "1.14.1" [deps.JSON3.weakdeps] ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + [[deps.LaTeXStrings]] git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.4.0" -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.6.0+0" - -[[deps.LibGit2]] -deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -version = "1.11.0" - -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.2+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.0+1" - [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" @@ -253,13 +237,13 @@ version = "1.11.0" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.11.0" +version = "1.12.0" [[deps.LogExpFunctions]] deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "a2d09619db4e765091ee5c6ffe8872849de0feea" +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.28" +version = "0.3.29" [deps.LogExpFunctions.extensions] LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" @@ -276,7 +260,7 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" [[deps.Markdown]] -deps = ["Base64"] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" version = "1.11.0" @@ -287,9 +271,10 @@ uuid = "739be429-bea8-5141-9913-cc70e7f3736d" version = "1.1.9" [[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8" uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.6+0" +version = "2.28.1010+0" [[deps.Missings]] deps = ["DataAPI"] @@ -303,7 +288,7 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.12.12" +version = "2025.5.20" [[deps.NATS]] deps = ["Base64", "BufferedStreams", "CodecBase", "Dates", "DocStringExtensions", "JSON3", "MbedTLS", "NanoDates", "Random", "ScopedValues", "Sockets", "Sodium", "StructTypes", "URIs"] @@ -319,51 +304,44 @@ version = "1.0.3" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" +version = "1.3.0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.27+1" +version = "0.3.29+0" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.5+0" +version = "0.8.7+0" [[deps.OpenSpecFun_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" -version = "0.5.5+0" +version = "0.5.6+0" [[deps.OrderedCollections]] -git-tree-sha1 = "12f1439c4f986bb868acda6ea33ebc78e19b95ad" +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.7.0" +version = "1.8.1" [[deps.PDMats]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "949347156c25054de2db3b166c52ac4728cbad65" +git-tree-sha1 = "d922b4d80d1e12c658da7785e754f4796cc1d60d" uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.31" +version = "0.11.36" +weakdeps = ["StatsBase"] + + [deps.PDMats.extensions] + StatsBaseExt = "StatsBase" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.1" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.11.0" - - [deps.Pkg.extensions] - REPLExt = "REPL" - - [deps.Pkg.weakdeps] - REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "2.8.3" [[deps.PooledArrays]] deps = ["DataAPI", "Future"] @@ -373,15 +351,15 @@ version = "1.4.3" [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.1" +version = "1.3.3" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.PrettyPrinting]] git-tree-sha1 = "142ee93724a9c5d04d78df7006670a93ed1b244e" @@ -389,10 +367,10 @@ uuid = "54e16d92-306c-5ea0-a30b-337be88ac337" version = "0.4.2" [[deps.PrettyTables]] -deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "c5a07210bd060d6a8491b0ccdee2fa0235fc00bf" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.4.0" +version = "3.1.2" [[deps.Printf]] deps = ["Unicode"] @@ -400,15 +378,15 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" [[deps.PtrArrays]] -git-tree-sha1 = "77a42d78b6a92df47ab37e177b2deac405e1c88f" +git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.2.1" +version = "1.3.0" [[deps.QuadGK]] deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "cda3b045cf9ef07a08ad46731f5a3165e56cf3da" +git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.11.1" +version = "2.11.2" [deps.QuadGK.extensions] QuadGKEnzymeExt = "Enzyme" @@ -416,6 +394,11 @@ version = "2.11.1" [deps.QuadGK.weakdeps] Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -428,9 +411,9 @@ version = "1.2.2" [[deps.Rmath]] deps = ["Random", "Rmath_jll"] -git-tree-sha1 = "852bd0f55565a9e973fcfee83a84413270224dc4" +git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" -version = "0.8.0" +version = "0.9.0" [[deps.Rmath_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -444,15 +427,15 @@ version = "0.7.0" [[deps.ScopedValues]] deps = ["HashArrayMappedTries", "Logging"] -git-tree-sha1 = "1147f140b4c8ddab224c94efa9569fc23d63ab44" +git-tree-sha1 = "c3b2323466378a2ba15bea4b2f73b081e022f473" uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" -version = "1.3.0" +version = "1.5.0" [[deps.SentinelArrays]] deps = ["Dates", "Random"] -git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" +git-tree-sha1 = "712fb0231ee6f9120e005ccd56297abbc053e7e0" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.7" +version = "1.4.8" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -470,20 +453,20 @@ version = "1.1.2" [[deps.SortingAlgorithms]] deps = ["DataStructures"] -git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.1" +version = "1.2.2" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.11.0" +version = "1.12.0" [[deps.SpecialFunctions]] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "2f5d4697f21388cbe1ff299430dd169ef97d7e14" +git-tree-sha1 = "f2685b435df2613e25fc10ad8c26dddb8640f547" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.4.0" +version = "2.6.1" [deps.SpecialFunctions.extensions] SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" @@ -503,21 +486,21 @@ weakdeps = ["SparseArrays"] [[deps.StatsAPI]] deps = ["LinearAlgebra"] -git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +git-tree-sha1 = "9d72a13a3f4dd3795a195ac5a44d7d6ff5f552ff" uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.7.0" +version = "1.7.1" [[deps.StatsBase]] -deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "5cf7606d6cef84b543b483848d4ae08ad9832b21" +deps = ["AliasTables", "DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "064b532283c97daae49e544bb9cb413c26511f8c" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.34.3" +version = "0.34.8" [[deps.StatsFuns]] deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] -git-tree-sha1 = "b423576adc27097764a90e163157bcfc9acf0f46" +git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "1.3.2" +version = "1.5.2" [deps.StatsFuns.extensions] StatsFunsChainRulesCoreExt = "ChainRulesCore" @@ -529,9 +512,9 @@ version = "1.3.2" [[deps.StringManipulation]] deps = ["PrecompileTools"] -git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" +git-tree-sha1 = "725421ae8e530ec29bcbdddbe91ff8053421d023" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.4.0" +version = "0.4.1" [[deps.StructTypes]] deps = ["Dates", "UUIDs"] @@ -539,6 +522,24 @@ git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" version = "1.11.0" +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "79529b493a44927dd5b13dde1c7ce957c2d049e4" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.6.0" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + [[deps.SuiteSparse]] deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" @@ -546,7 +547,7 @@ uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.7.0+0" +version = "7.8.3+2" [[deps.TOML]] deps = ["Dates"] @@ -561,14 +562,9 @@ version = "1.0.1" [[deps.Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] -git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" +git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.12.0" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" +version = "1.12.1" [[deps.TranscodingStreams]] git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" @@ -576,9 +572,9 @@ uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.11.3" [[deps.URIs]] -git-tree-sha1 = "24c1c558881564e2217dcf7840a8b2e10caeb0f9" +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.6.0" +version = "1.6.1" [[deps.UUIDs]] deps = ["Random", "SHA"] @@ -603,25 +599,15 @@ version = "1.6.1" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+1" +version = "1.3.1+2" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.11.0+0" +version = "5.15.0+0" [[deps.libsodium_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f76d682d87eefadd3f165d8d9fda436464213142" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" -version = "1.0.20+3" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.59.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+2" +version = "1.0.21+0" diff --git a/Project.toml b/Project.toml index 5a68284..b8b5e5d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -authors = ["tonaerospace "] version = "0.3.1" +authors = ["tonaerospace "] [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" @@ -9,7 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" -JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -17,4 +17,5 @@ SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] +JSON = "1.3.0" NATS = "0.1.0" diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d0b3e2 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +Todo: + - [WORKING] update with JSON + + Change from previous version: + - replace JSON3 with JSON \ No newline at end of file diff --git a/src/GeneralUtils.jl b/src/GeneralUtils.jl index 41463b1..5addf11 100644 --- a/src/GeneralUtils.jl +++ b/src/GeneralUtils.jl @@ -24,16 +24,7 @@ using .interface #------------------------------------------------------------------------------------------------100 -""" version 0.0.4 - Todo: - - [*1] cartesianAssign for different matrix dimension - Change from version: 0.0.3 - - - - All features - -""" diff --git a/src/communication.jl b/src/communication.jl index f44f77f..63894ff 100644 --- a/src/communication.jl +++ b/src/communication.jl @@ -5,10 +5,10 @@ export generate_msgMeta # sendMqttMsg, sendReceiveMqttMsg, mqttClientInstance, mqttClientInstance_v2, # dataTransferOverMQTT_sender, dataTransferOverMQTT_receiver -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, +using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, SHA, PrettyPrinting using ..util - +#[PENDING] update code to use JSON # ---------------------------------------------- 100 --------------------------------------------- # # abstract type mqttClientInstance end diff --git a/src/dbUtil.jl b/src/dbUtil.jl index 2e4fea8..e5f6b08 100644 --- a/src/dbUtil.jl +++ b/src/dbUtil.jl @@ -2,10 +2,10 @@ module dbUtil export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, +using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, SHA using ..util - +#[PENDING] update code to use JSON # ---------------------------------------------- 100 --------------------------------------------- # diff --git a/src/interface.jl b/src/interface.jl index dea5dcb..2a41697 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -8,9 +8,9 @@ export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!, getDictPath, detectKeywordVariation, textToDict -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV +using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV using ..util, ..communication - +#[WORKING] update code to use JSON # ---------------------------------------------- 100 --------------------------------------------- # noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) diff --git a/src/llmUtil.jl b/src/llmUtil.jl index a021dd9..7c11afd 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -2,11 +2,11 @@ module llmUtil export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection, deFormatLLMtext, extractthink -using UUIDs, JSON3, Dates +using UUIDs, JSON, Dates using GeneralUtils # ---------------------------------------------- 100 --------------------------------------------- # - +#[PENDING] update code to use JSON """ Convert a single chat dictionary into LLM model instruct format. diff --git a/src/util.jl b/src/util.jl index 671a201..3ad8496 100644 --- a/src/util.jl +++ b/src/util.jl @@ -9,7 +9,7 @@ export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, rep extractTextBetweenCharacter, extractTextBetweenString, convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex -using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames +using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames # ---------------------------------------------- 100 --------------------------------------------- # @@ -378,7 +378,7 @@ end # Example ```jldoctest - julia> using DataFrames, JSON3, GeneralUtils + julia> using DataFrames, GeneralUtils julia> df = DataFrame(A = [1, 2, 3], B = ["apple", "banana", "cherry"]) julia> vectorDict = GeneralUtils.dfToVectorDict(df) [Dict{String, Any}("B" => "apple", "A" => 1), @@ -416,7 +416,7 @@ end # Example ```jldoctest - julia> using GeneralUtils, Dates, JSON3, UUIDs + julia> using GeneralUtils, Dates, UUIDs julia> vecDict = [Dict("a" => i) for i in 1:10] julia> d = GeneralUtils.disintegrate_vectorDict(vecDict, 3) julia> println(d[:data]) @@ -568,7 +568,7 @@ end # Example ```jldoctest - julia> using DataFrames + julia> using DataFrames, GeneralUtils julia> df = DataFrame(name=["Alice", "Bob"], age=[25, 30]) 2×2 DataFrame @@ -578,7 +578,7 @@ end │ 1 │ Alice 25 │ 2 │ Bob 30 - julia> dataframe_to_json_list(df) + julia> GeneralUtils.dataframe_to_json_list(df) 2-element Vector{String}: "{\"name\":\"Alice\",\"age\":25}" "{\"name\":\"Bob\",\"age\":30}" From 170b0bad15aba1f8a880cb3d389f8bd107744cfa Mon Sep 17 00:00:00 2001 From: narawat Date: Sat, 22 Nov 2025 10:45:08 +0700 Subject: [PATCH 09/10] update --- Manifest.toml | 67 +++++++++++++++++++++++++++++++++++++++++++++++- Project.toml | 2 ++ src/interface.jl | 38 ++++++++++++++++++++------- 3 files changed, 97 insertions(+), 10 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 39d3cc8..bbc87ae 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.1" manifest_format = "2.0" -project_hash = "8a1366f01fdc86b8cec7588b2642a428f8711411" +project_hash = "c7a2fd47754abe433f089d6232e90fcb82c87f85" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -29,6 +29,12 @@ git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" version = "0.10.15" +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "9ce926a33a8608421a4d45c012884165b3fcd3ee" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "2.0.2" + [[deps.CodecBase]] deps = ["TranscodingStreams"] git-tree-sha1 = "40956acdbef3d8c7cc38cba42b56034af8f8581a" @@ -51,6 +57,11 @@ weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" +[[deps.Compiler]] +git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" +version = "0.1.1" + [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" @@ -123,6 +134,10 @@ version = "0.9.24" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + [[deps.FillArrays]] deps = ["LinearAlgebra"] git-tree-sha1 = "5bfcd42851cf2f1b303f51525a54dc5e98d408a3" @@ -220,6 +235,12 @@ version = "1.14.3" [deps.JSON3.weakdeps] ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +git-tree-sha1 = "b6c76964c65ebf8309460fb8f0f437b4a59d809b" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.10.7" + [[deps.JuliaSyntaxHighlighting]] deps = ["StyledStrings"] uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" @@ -230,6 +251,21 @@ git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.4.0" +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" @@ -259,6 +295,12 @@ version = "0.3.29" uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" +[[deps.LoweredCodeUtils]] +deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] +git-tree-sha1 = "e24491cb83551e44a69b9106c50666dea9d953ab" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +version = "3.4.4" + [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -316,6 +358,11 @@ deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.7+0" +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.1+0" + [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" @@ -409,6 +456,24 @@ git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "1.2.2" +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.Revise]] +deps = ["CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] +git-tree-sha1 = "85d94c2be31f58728cd69d13f2e0bdd7ecf6dfe9" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.12.1" + + [deps.Revise.extensions] + DistributedExt = "Distributed" + + [deps.Revise.weakdeps] + Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + [[deps.Rmath]] deps = ["Random", "Rmath_jll"] git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" diff --git a/Project.toml b/Project.toml index b8b5e5d..c25f415 100644 --- a/Project.toml +++ b/Project.toml @@ -13,9 +13,11 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] JSON = "1.3.0" NATS = "0.1.0" +Revise = "3.12.1" diff --git a/src/interface.jl b/src/interface.jl index 2a41697..ba15b2f 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -187,23 +187,24 @@ end # ---------------------------------------------- 100 --------------------------------------------- # -""" Array_to_JSON3_str(data::AbstractArray) - encode Array to JSON3 String +""" Array_to_JSON_str(data::AbstractArray) + + encode Array to JSON String # Example a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] - json3_str = Array_to_JSON3_str(a) - - json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} + jsonStr = Array_to_JSON_str(a) + jsonStr = "{\"Array\":[[1.23,9987.1,-0.0027],[4.7889,-123.07,-6.75]],\"size\":[3,2]}" """ -function Array_to_JSON3_str(data::AbstractArray) - d = Dict("Array"=> data, "size"=>size(data)) - json3_str = JSON3.write(d) - return json3_str +function Array_to_JSON_str(data::AbstractArray) + d = Dict("Array"=> data, "size"=>size(data)) + jsonStr = JSON.json(d) + return jsonStr end + # ---------------------------------------------- 100 --------------------------------------------- # """ JSON3_str_to_Array(json3_str::String) @@ -223,6 +224,25 @@ function JSON3_str_to_Array(json3_str::String) return array end +#[WORKING] +""" JSON_str_to_Array(json_str::String) + + decode JSON String to Array + + # Example + + jsonStr = "{\"Array\":[[1.23,9987.1,-0.0027],[4.7889,-123.07,-6.75]],\"size\":[3,2]}" + a = JSON_str_to_Array(jsonStr) + +""" +function JSON_str_to_Array(jsonStr::String) + jsonObj = JSON.parse(jsonStr) + array = reshape(Array(jsonObj.Array), (jsonObj.size[1], jsonObj.size[2])) + return array +end + + + # ---------------------------------------------- 100 --------------------------------------------- # """ Convert JSON3.read object to OrderedDict From 35c2b4c21164324095fe74973028beb5c84bdb28 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 17 Dec 2025 12:39:18 +0700 Subject: [PATCH 10/10] use JSON instead of JSON3 --- src/interface.jl | 212 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 171 insertions(+), 41 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index ba15b2f..a7b45eb 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -6,11 +6,11 @@ export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax, multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween, isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!, - getDictPath, detectKeywordVariation, textToDict + getDictPath, detectKeywordVariation, textToDict, dictify, ordereddictify using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV using ..util, ..communication -#[WORKING] update code to use JSON + # ---------------------------------------------- 100 --------------------------------------------- # noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) @@ -148,7 +148,7 @@ function findMax(collection::AbstractVector) return maxValue, maxIndex, matchPosition end - +# ---------------------------------------------- 100 --------------------------------------------- # """ read_textfile_by_index(folder_path::String, read_file_number::Integer=1) @@ -185,8 +185,94 @@ function read_textfile_by_index(folder_path::String, read_file_number::Integer=1 end -# ---------------------------------------------- 100 --------------------------------------------- # +""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary. + +# What it does + - Walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`, + `Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where + every dictionary-like node is a plain `Dict` and every array-like node is a + `Vector{Any}`. Scalar values (numbers, strings, booleans, `nothing`, etc.) + are returned unchanged. + - Does **not** mutate the input; it always allocates new containers. + +# Arguments +- `x` + Any Julia value. If `x` is an `AbstractDict` it will be converted to a `Dict`; + if it is an `AbstractArray` its elements will be processed recursively. + +# Keyword Arguments +- `stringkey::Bool=false` + If `true`, every dictionary key is converted to `String` via `string(k)`. + If `false`, original key objects are preserved (useful when keys are already + `String`, `Symbol`, or other types you want to keep). + +# Return +- A newly allocated nested structure composed of `Dict{Any,Any}` and + `Vector{Any}` that mirrors the input shape but uses plain Julia containers. + +# Notes +- The function treats any `AbstractDict` as a mapping source, so it works with + `JSON.Object`, `Dict`, `OrderedDict`, etc. +- Arrays are returned as `Vector{Any}` with their elements processed + recursively. + +# Examples +```jldoctest +julia> using JSON +julia> d = Dict( + "a" => 4, + "b" => 6, + "c" => Dict( + "d"=>7, + :e=>Dict( + "f"=>"hey", + "g"=>Dict( + "world"=>[1, "2", 3, Dict(:dd=>4.7)] + ) + ) + ) + ) + +julia jsonstring = JSON.json(d) +julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object +julia> A2 = dictify(A1) +Dict{Any,Any} with 3 entries: + "a" => 4 + "b" => 6 + "c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7]))) + +julia> A3 = dictify(A1; stringkey=false) # preserves original key objects +julia> B1 = dictify(d; stringkey=true) # convert all keys in to string +Dict{Any, Any} with 3 entries: + "c" => Dict{Any, Any}("e"=>Dict{Any, Any}("f"=>"hey", "g"=>Dict{Any, Any}("world"=>Any[1, "2", 3, Dict{Any, Any}("dd"=>4.7)])), "d"=>7) + "b" => 6 + "a" => 4 +``` +""" +function dictify(x; stringkey::Bool=false) + # Dict-like objects + if x isa AbstractDict + # choose output key type container (String keys when requested) + out = Dict{Any,Any}() + for (k,v) in x + newk = stringkey ? string(k) : k + out[newk] = dictify(v; stringkey=stringkey) + end + return out + # Arrays / vectors: map elements recursively and return a Vector{Any} + elseif x isa AbstractArray + return [dictify(element; stringkey=stringkey) for element in x] + # everything else: return as-is (primitives, numbers, strings, etc.) + else + return x + end +end + + + + +# ---------------------------------------------- 100 --------------------------------------------- # """ Array_to_JSON_str(data::AbstractArray) @@ -204,27 +290,8 @@ function Array_to_JSON_str(data::AbstractArray) return jsonStr end - # ---------------------------------------------- 100 --------------------------------------------- # -""" JSON3_str_to_Array(json3_str::String) - - decode JSON3 String to Array - - # Example - - json3_str = {"Array":[1.23,9987.1,-0.0027,4.7889,-123.07,-6.75],"size":[3,2]} - a = JSON3_str_to_Array(json3_str) - - a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75] -""" -function JSON3_str_to_Array(json3_str::String) - d = JSON3.read(json3_str) - array = reshape(Array(d.Array), (d.size[1], d.size[2])) - return array -end - -#[WORKING] """ JSON_str_to_Array(json_str::String) decode JSON String to Array @@ -237,33 +304,95 @@ end """ function JSON_str_to_Array(jsonStr::String) jsonObj = JSON.parse(jsonStr) - array = reshape(Array(jsonObj.Array), (jsonObj.size[1], jsonObj.size[2])) + a = Array(jsonObj.Array) + array = hcat(a...) return array end - - # ---------------------------------------------- 100 --------------------------------------------- # -""" Convert JSON3.read object to OrderedDict +""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary. - # Example - dict = dictionary(["a"=>4, "b"=>6]) - OrDict = OrderedDict(dict) - jsonString = JSON3.write(OrDict) # use jsonString to exchange. One can save it to file or send it thru pub/sub - jsonObject = JSON3.read(jsonString) - OrDict2 = JSON3read_to_OrDict(jsonObject) # example here - Adict2 = dictionary(OrDict2) +# What it does + - Walks any nested structure composed of AbstractDict (e.g., JSON.Object, + Dict, OrderedDict) and AbstractArray and produces a new tree where + every dictionary-like node is an OrderedDict{Any,Any} and every array-like + node is a Vector{Any}. Scalar values (numbers, strings, booleans, + nothing, etc.) are returned unchanged. + - Does **not** mutate the input; it always allocates new containers. - Andyferris's github https://github.com/andyferris/Dictionaries.jl +# Arguments +- `x` + Any Julia value. If x is an AbstractDict it will be converted to an + OrderedDict{Any,Any}. if it is an AbstractArray its elements will be + processed recursively. + +# Keyword Arguments +- `stringkey::Bool=false` + If `true`, every dictionary key is converted to `String` via `string(k)`. + If `false`, original key objects are preserved (useful when keys are already + `String`, `Symbol`, or other types you want to keep). + +# Return +- A newly allocated nested structure composed of `OrderedDict{Any,Any}` and + `Vector{Any}` that mirrors the input shape but uses ordered Julia containers. + +# Notes +- The function treats any `AbstractDict` as a mapping source, so it works with + `JSON.Object`, `Dict`, `OrderedDict`, etc. +- Arrays are returned as `Vector{Any}` with their elements processed recursively. + +# Examples +```jldoctest +julia> using JSON +julia> d = Dict( + "a" => 4, + "b" => 6, + "c" => Dict( + "d"=>7, + :e=>Dict( + "f"=>"hey", + "g"=>Dict( + "world"=>[1, "2", 3, Dict(:dd=>4.7)] + ) + ) + ) + ) + +julia jsonstring = JSON.json(d) +julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object +julia> A2 = OrderedDict(A1) +Dict{Any,Any} with 3 entries: + "a" => 4 + "b" => 6 + "c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7]))) + +julia> A3 = OrderedDict(A1; stringkey=false) # preserves original key objects +julia> B1 = OrderedDict(d; stringkey=true) # convert all keys in to string +Dict{Any, Any} with 3 entries: + "c" => Dict{Any, Any}("e"=>Dict{Any, Any}("f"=>"hey", "g"=>Dict{Any, Any}("world"=>Any[1, "2", 3, Dict{Any, Any}("dd"=>4.7)])), "d"=>7) + "b" => 6 + "a" => 4 +``` +Ref. https://github.com/andyferris/Dictionaries.jl """ -function JSON3read_to_OrDict(x) - dict = OrderedDict() - for (k, v) in x - k = string(k) - dict[k] = v +function ordereddictify(x; stringkey::Bool=false) + # Dict-like objects + if x isa AbstractDict + # choose output key type container (String keys when requested) + out = OrderedDict{Any,Any}() + for (k,v) in x + newk = stringkey ? string(k) : k + out[newk] = ordereddictify(v; stringkey=stringkey) + end + return out + # Arrays / vectors: map elements recursively and return a Vector{Any} + elseif x isa AbstractArray + return [ordereddictify(element; stringkey=stringkey) for element in x] + # everything else: return as-is (primitives, numbers, strings, etc.) + else + return x end - return dict end #------------------------------------------------------------------------------------------------100 @@ -1227,6 +1356,7 @@ Detects if a keyword exists in the text in different case variations (lowercase, julia> detectKeywordVariation("missing", "complete data") nothing + ``` """ function detectKeywordVariation(keyword::String, text::String)::Union{Nothing, Array{String}} # Define the keyword variations to search for