module interface export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, replaceMoreThan, replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul3Dto3DmanyTo1batch, matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax, multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween, isLess, allTrue, getStringBetweenCharacters, mkDictPath!, dict_to_string_html, getDictPath, detectKeywordVariation, textToDict, dictify, ordereddictify using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV using ..util, ..communication # ---------------------------------------------- 100 --------------------------------------------- # noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a) findNotZero(x::AbstractVector) = findall( (!iszero).(x) ) replaceMoreThan(x, target, replaceValue) = x > target ? replaceValue : x replaceMoreThan(x, target, a, b) = x > target ? a : b replaceLessThan(x, target, replaceValue) = x < target ? replaceValue : x replaceLessThan(x, target, a, b) = x < target ? a : b replaceBetween(x, lowerbound, upperbound, replaceValue) = lowerbound < x < upperbound ? replaceValue : x precision(x::Array{<:Array}) = ( std(mean.(x)) / mean(mean.(x)) ) * 100 precision(x::Array) = std(x) / mean(x) * 100 replaceAt!(x::AbstractVector, ind::Number, value::Number) = x[ind] = value notZero(x::AbstractVector) = (!iszero).(x) Zero(x::AbstractVector) = iszero.(x) isNan(x::AbstractVector) = isnan.(x) isInf(x::Number) = abs(x) === Inf isInf(x::AbstractVector) = isinf.(x) isNotEqual(x::Number, target::Number) = isequal(isequal(x, target), 0) isBetween(x, lowerlimit, upperlimit) = lowerlimit < x < upperlimit ? true : false absolute(x::AbstractVector) = abs.(x) vecEleMul(x::AbstractVector, y::AbstractVector) = x .* y vecEleMul(x::Number, y::AbstractVector) = x .* y expDecay(initialValue::Number, decayFactor::Number, timePass::Number) = initialValue * (1 - decayFactor)^timePass mul!(x::AbstractVector, y::AbstractVector) = x .*= y mul(x::AbstractVector, y::AbstractVector) = x .* y allTrue(args...) = false ∈ [args...] ? false : true ReLu(x::Number) = max(0, x) updateVector!(x::AbstractVector, target::Number) = x .= target updateVector!(x::AbstractVector, target::AbstractArray) = x .= target function selectAdd!(x::AbstractVector, ind::AbstractVector, value::AbstractVector) @. x = x + (ind * value) end """ FindIndex(input::String, target::Char) Arguments: text, input text target, target character Return: (a bool vector of match/not match, position vector of the matched) Example: ```jldoctest julia> using GeneralUtils julia> text = "Hello World!" julia> findIndex(text, 'l') (Bool[0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0], [3, 4, 10]) ``` """ function findIndex(text::String, target::Char) charlist = [i for i in text] match_position = isequal.(charlist, target) match_index = findall(isequal.(match_position, 1)) return match_position, match_index end function findIndex(input::Array, target::Number) match_position = isequal.(input, target) match_index = findall(match_position) return match_position, match_index end # function findIndex(input::Array, target::Array) # match_position = isone.(zeros(length(input))) # for i in target # match_position = match_position + isequal.(input, i) # end # match_position = replaceMoreThan.(match_position, 1) # match_index = findall(isone.(match_position)) # Findall donot work with Int64 vector [1, 0, 0, 1]. # # It only works with BitVector. isone() converts Int64 vector [1, 0, 0, 1] into # # BitVector [1, 0, 0, 1] # return match_position, match_index # end function findIndex(input::Array, target::Symbol) match_position = isequal.(input, target) match_index = findall(match_position) return match_position, match_index end function findIndex(collection::Array{String}, target::String) match_position = isequal.(collection, target) match_index = findall(match_position) return match_position, match_index end function findIndex(collection::Array{String}, target::Array{String}) match_position = nothing match_index = nothing for i in target match_pos = isequal.(collection, i) match_ind = findall(match_pos) if match_position === nothing match_position = match_pos else match_position = hcat(match_position, match_pos) end if match_index === nothing match_index = match_ind else match_index = hcat(match_index, match_ind) end end return match_position, match_index end function findIndex(collection::OrderedDict, target::Symbol) collection_keys = keys(collection) collection_keys_array = [i for i in collection_keys] match_position = isequal.(collection_keys_array, target) match_index = findall(match_position) return match_position, match_index end function findMax(collection::AbstractVector) maxValue, maxIndex = findmax(collection) matchPosition = isequal.(collection, maxValue) return maxValue, maxIndex, matchPosition end # ---------------------------------------------- 100 --------------------------------------------- # """ Reads the x-th text file from a folder, where files are listed by the OS without explicit sorting. Returns the file number, filename, and content. # Arguments - `folder_path::String` Path to the folder containing text files. - `read_file_number::Integer=1` Which file to read (1-based index). Defaults to the first file. # Return - A tuple of `(file_number::Integer, filename::String, content)` where: - `file_number` is the index of the file that was read - `filename` is the actual filename string - `content` is a vector of lines from the file (or `nothing` if error) # Notes - Errors if `folder_path` is not a valid directory. - Errors if `read_file_number` exceeds the number of files in the folder. # Examples ```jldoctest julia> using GeneralUtils julia> result = read_textfile_by_index("/path/to/folder", 2) (2, "sample.txt", ["line1", "line2", ...]) ``` """ function read_textfile_by_index(folder_path::String, read_file_number::Integer=1) if isdir(folder_path) filenumber = length(readdir(folder_path)) if read_file_number > filenumber error("you specified read_file_number = $read_file_number which is out of range, the cleaned data folder has only $filenumber files") return nothing, nothing, nothing else content = 0 # open each file in the directory and read filename = readdir(folder_path, join=true, sort=false)[read_file_number] f = open(filename) content = readlines(f) # content = read(f) close(f) end return read_file_number, filename, content else error("ERROR no file or folder at $folder_path") return nothing, nothing, nothing end end """ Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary. The function walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`, `Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where every dictionary-like node is a plain `Dict` and every array-like node is a `Vector{Any}`. Scalar values (numbers, strings, booleans, `nothing`, etc.) are returned unchanged. Does **not** mutate the input; it always allocates new containers. # Arguments - `x` Any Julia value. If `x` is an `AbstractDict` it will be converted to a `Dict`; if it is an `AbstractArray` its elements will be processed recursively. # Keyword Arguments - `keytype::Type=Any` The key type for the output Dict. Use `String` for `Dict{String,Any}`, `Symbol` for `Dict{Symbol,Any}`, or `Any` to preserve original key types. - `stringkey::Bool=false` If `true`, every dictionary key is converted to `String` via `string(k)`. This parameter is ignored when `keytype` is explicitly set. # Return - A newly allocated nested structure composed of `Dict{keytype,Any}` and `Vector{Any}` that mirrors the input shape but uses plain Julia containers. # Notes - The function treats any `AbstractDict` as a mapping source, so it works with `JSON.Object`, `Dict`, `OrderedDict`, etc. - Arrays are returned as `Vector{Any}` with their elements processed recursively. # Examples ```jldoctest julia> using JSON julia> d = Dict( "a" => 4, "b" => 6, "c" => Dict( "d"=>7, :e=>Dict( "f"=>"hey", "g"=>Dict( "world"=>[1, "2", 3, Dict(:dd=>4.7)] ) ) ) ) julia jsonstring = JSON.json(d) julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object julia> A2 = dictify(A1; keytype=String) Dict{String,Any} with 3 entries: "a" => 4 "b" => 6 "c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7]))) julia> A3 = dictify(A1; keytype=Symbol) Dict{Symbol,Any} with 3 entries: :a => 4 :b => 6 :c => Dict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7]))) julia> B1 = dictify(d; keytype=String) Dict{String, Any} with 3 entries: """ function dictify(x; keytype::Type=Any) # Dict-like objects if x isa AbstractDict # choose output key type container out = Dict{keytype,Any}() for (k,v) in x if keytype === String newk = string(k) elseif keytype === Symbol newk = Symbol(string(k)) else newk = k end out[newk] = dictify(v; keytype=keytype) end return out # Arrays / vectors: map elements recursively and return a Vector{Any} elseif x isa AbstractArray return [dictify(element; keytype=keytype) for element in x] # everything else: return as-is (primitives, numbers, strings, etc.) else return x end end # ---------------------------------------------- 100 --------------------------------------------- # """ Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary. The function walks any nested structure composed of AbstractDict (e.g., JSON.Object, Dict, OrderedDict) and AbstractArray and produces a new tree where every dictionary-like node is an OrderedDict{Any,Any} and every array-like node is a Vector{Any}. Scalar values (numbers, strings, booleans, nothing, etc.) are returned unchanged. Does **not** mutate the input; it always allocates new containers. # Arguments - `x` Any Julia value. If x is an AbstractDict it will be converted to an OrderedDict{Any,Any}. if it is an AbstractArray its elements will be processed recursively. # Keyword Arguments - `keytype::Type=Any` The key type for the output Dict. Use `String` for `OrderedDict{String,Any}`, `Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types. # Return - A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and `Vector{Any}` that mirrors the input shape but uses ordered Julia containers. # Notes - The function treats any `AbstractDict` as a mapping source, so it works with `JSON.Object`, `Dict`, `OrderedDict`, etc. - Arrays are returned as `Vector{Any}` with their elements processed recursively. # Examples ```jldoctest julia> using JSON julia> d = Dict( "a" => 4, "b" => 6, "c" => Dict( "d"=>7, :e=>Dict( "f"=>"hey", "g"=>Dict( "world"=>[1, "2", 3, Dict(:dd=>4.7)] ) ) ) ) julia jsonstring = JSON.json(d) julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object julia> A2 = ordereddictify(A1; keytype=String) OrderedDict{String,Any} with 3 entries: "a" => 4 "b" => 6 "c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7]))) julia> A3 = ordereddictify(A1; keytype=Symbol) OrderedDict{Symbol,Any} with 3 entries: :a => 4 :b => 6 :c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7]))) julia> B1 = ordereddictify(d; keytype=String) OrderedDict{String, Any} with 3 entries: "c" => OrderedDict{String, Any}("e"=>OrderedDict{String, Any}("f"=>"hey", "g"=>OrderedDict{String, Any}("world"=>Any[1, "2", 3, OrderedDict{String, Any}("dd"=>4.7)])), "d"=>7) "b" => 6 "a" => 4 ``` Ref. https://github.com/andyferris/Dictionaries.jl """ function ordereddictify(x; keytype::Type=Any) # Dict-like objects if x isa AbstractDict # choose output key type container out = OrderedDict{keytype,Any}() for (k,v) in x if keytype === String newk = string(k) elseif keytype === Symbol newk = Symbol(string(k)) else newk = k end out[newk] = ordereddictify(v; keytype=keytype) end return out # Arrays / vectors: map elements recursively and return a Vector{Any} elseif x isa AbstractArray return [ordereddictify(element; keytype=keytype) for element in x] # everything else: return as-is (primitives, numbers, strings, etc.) else return x end end #----------------------------------------------100--------------------------------------------- """ print time of cpu executtion at the line inwhich this macro is used """ macro timeline(expr) quote print("line ", $(__source__.line), ": ") @time $(esc(expr)) end end batchindex(batch_counter::Number, batch_size::Number; offset=0) = (offset + (batch_counter-1) * batch_size + 1) : offset + (batch_counter * batch_size) function flip_true_false(x::Bool) if x == true x = false elseif x == false x = true else error("undefined condition line $(@__LINE__)") end return x end function flip_true_false(x::Int) if x == 1 x = 0 elseif x == 0 x = 1 else throw("not define input of type $(typeof(x)) yet") end return x end """ Return drawed index # Example drawed_index = randomWithProb([0.5, 0.2, 0.3]) """ randomWithProb(probability::AbstractVector) = rand(Distributions.Categorical(probability)) # return drawed index """ Draw from choices according to its probability. Probability range is 0.0 to 1.0 and all probability must summed up to 1 (may get probability from NNlib's softmax function) # Example draw = randomChoiceWithProb([true, false, nothing], [0.5, 0.2, 0.3]) """ function randomChoiceWithProb(choices::Array, probability::Array) if length(choices) != length(probability) error("random is not possible, choices array length != probability array length") elseif sum(probability) != 1.0 error("probability does not sum to 1.0") end return choices[randomWithProb(probability)] end function randomChoiceOnTarget(target::Number, targetMatch::Number, choices::AbstractVector, probability::AbstractVector) if length(choices) != length(probability) throw("random is not possible, choices array length != probability array length") end return target == targetMatch ? randomChoiceWithProb(choices, probability) : target # dist = Distributions.Categorical(probability) # draw_result = choices[rand(dist)] end function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector, probability::AbstractVector) return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,)) end """ Compute the linearly weighted average of an array. The function assigns weights proportional to position indices (1, 2, 3, ...) to array elements and returns the weighted average. # Arguments - `a::Array` Array of numeric values. Elements must support multiplication with numbers and summation. # Return - The linearly weighted average as a floating-point number. # Formula For an array `a` with `n` elements, computes: ``` sum(i * a[i]) / sum(a) for i = 1 to n ``` # Examples ```jldoctest julia> using GeneralUtils julia> a = [10, 20, 30] julia> linearly_weighted_avg(a) 23.333333333333332 ``` """ function linearly_weighted_avg(a::Array) total = 0.0 for (i, v) in enumerate(a) total = total + (i * v) end return total / sum(a) end """ Convert a variable's value (String) into a Symbol. The function takes a variable containing a String value and converts it to a Symbol using Julia's expression interpolation mechanism. # Arguments - `variable` Any variable whose value is a String. The function uses `string(variable)` internally to obtain the value. # Return - A `Symbol` constructed from the string value of the input variable. # Notes - This function uses variable interpolation to capture the variable's value as a Symbol. It works with any variable type that can be converted to String. # Examples ```jldoctest julia> using GeneralUtils julia> x = "hello" julia> variable_str_to_symbol(x) :hello julia> y = "world_test" julia> variable_str_to_symbol(y) :world_test ``` """ function variable_str_to_symbol(variable) semi = :($variable) symbol = Symbol(semi) return symbol end """ get useable type of specified fieldname inside a composite struct # Example julia> @Base.kwdef mutable struct some_struct a::Union{Bool, Nothing} = nothing b::Union{Float64, Nothing} = nothing c::Union{Int64, AbstractFloat} = 3.5 d::Union{String, Nothing} = nothing end julia> a = some_struct() julia> fieldname_useable_type(some_struct, :c) =result=> [Int64, Float64] """ function fieldname_useable_type(somestruct, fieldname::Symbol; test_types=[2.0, 2, true, :h, "str", 'c', missing, nothing])::Vector{DataType} new_instance = somestruct() useable_type = [] for i in test_types try new_instance.:($fieldname) = i type = typeof(new_instance.:($fieldname)) if type ∉ useable_type push!(useable_type, type) end catch end end return useable_type end """ Draw unique elements from a list without replacement. The function randomly selects a specified number of distinct elements from a collection, optionally excluding certain elements from consideration. Uses in-place shuffling for efficiency. # Arguments - `drawOptions::Array` Collection of elements to draw from. - `draw_number::Integer` Number of unique elements to draw. # Keyword Arguments - `exclude_list::Union{AbstractArray,Nothing}=nothing` Elements to exclude from the drawing pool. If `nothing`, no elements are excluded. # Return - An array of `draw_number` unique elements drawn from `drawOptions`, excluding any elements in `exclude_list`. # Notes - The function copies `drawOptions` and shuffles in-place, then pops elements sequentially to ensure uniqueness. - Errors if `draw_number` exceeds the number of available elements after exclusion. # Examples ```jldoctest julia> using GeneralUtils julia> options = [1, 2, 3, 4, 5] julia> randomNoRepeat(options, 3) [3, 1, 5] julia> randomNoRepeat(options, 2; exclude_list=[1, 5]) [4, 2] ``` """ function randomNoRepeat(drawOptions::Array, draw_number::Integer; exclude_list::Union{AbstractArray,Nothing}=nothing) draw_option = copy(drawOptions) draw_option = isnothing(exclude_list) ? draw_option : filter!(x -> x ∉ exclude_list, draw_option) shuffle!(draw_option) drawed_items = [] while length(drawed_items) < draw_number push!(drawed_items, pop!(draw_option)) end return drawed_items end """ using cron to schedule backup job by 1. sudo nano /etc/crontab <<< this is a system-wide cron file 2. to execute julia file @ 2.00am everyday add the following line at the buttom of the file 0 2 * * * root julia-1.7 /home/syncthing_backup_script.jl Requirements using Dates """ function folderBackup(sourceFolderAbsolutePath::String, # absolute path to folder to be backuped backupFolderAbsolutePath::String; # absolute path to folder used to store backup file totalBackupFiles::Integer=7, # total backup file, the oldest will be deleted containerName::Union{Array{String}, Nothing}=nothing) # container using source_folder sep = (Sys.iswindows() ? "\\" : '/') if sourceFolderAbsolutePath[end] == sep sourceFolderAbsolutePath = sourceFolderAbsolutePath[1:end-1] end if backupFolderAbsolutePath[end] != sl backupFolderAbsolutePath = backupFolderAbsolutePath * sep end if isdir(backupFolderAbsolutePath) else mkpath(backupFolderAbsolutePath) end # stop running docker container service if containerName !== nothing println("stop running services") for i in containerName try run(`docker stop $i`) catch; end sleep(10) # wait for services to stop end end # do backup println("doing backup now") timestamp = string(Dates.now()) name = split(sourceFolderAbsolutePath, sep)[end] * "--" filename = name * timestamp * ".zip" # resulting compressed filename run(`chmod -R a+rwx $sourceFolderAbsolutePath`) # zip -r [destination+filename] [source folder to be zipped] run(`zip -r $(backupFolderAbsolutePath * filename) $sourceFolderAbsolutePath`) # check if total backup file is more than user specified, if yes, delete the oldest backup backupFiles = readdir(backupFolderAbsolutePath) while length(backupFiles) > totalBackupFiles run(`rm $(backupFolderAbsolutePath * backupFiles[1])`) backupFiles = readdir(backupFolderAbsolutePath) end # start docker services if containerName !== nothing println("start services") for i in containerName try run(`docker start $i`) catch; end sleep(10) # wait for services to stop end end end function lowerclip!(data::AbstractVector, lowerbound::Number) replace!(x -> x < lowerbound ? lowerbound : x, data) end function upperclip!(data::AbstractVector, upperbound::Number) replace!(x -> x > upperbound ? upperbound : x, data) end function normalise(x::AbstractArray, mu, std) ϵ = oftype(x[1], 1e-5) μ = mu # σ = std(x, dims=dims, mean=μ, corrected=false) # use this when Zygote#478 gets merged σ = std return (x .- μ) ./ (σ .+ ϵ) end function minMaxScaler(x::AbstractVector) min = findmin(x)[1] max = findmax(x)[1] scaler(a::Number, min::Number, max::Number) = (a-min) / (max-min) return scaler.(x, min, max) end """ a = [-1e200, -1e-200, 1e200, 1e-200] \n result = vtclamp.(a, 1e-6, 1e6, -1e6, -1e-6) """ function customclamp(x::Number, poslo::Number, poshi::Number, neglo::Number, neghi::Number) signx = sign(x) if signx == -1 if neghi < x < 0 return neghi elseif x < neglo return neglo else return x end elseif signx == +1 if poshi < x return poshi elseif 0 < x < poslo return poslo else return x end end end function unitVec(x::AbstractVector) y = √(sum(x.^2)) return x./y end function replaceAt!(x::AbstractVector, ind::AbstractVector, value::Number) for i in ind x[i] = value end end function signbitVec(x::AbstractVector) sign = signbit.(x) * 1 signVec = replace(s -> s == 0 ? -1 : s, sign) return signVec end function deleteall!(x::AbstractVector) for i in 1:length(x) deleteat!(x, 1) end end """ Select specific range of vectors in a dict, return a new dict # Example dict = Dict(:a => [1:5...], :b => [6:10...]) call -> selectRange(dict, 1:3) return -> Dict{Any, Any} with 2 entries: :a => [1, 2, 3] :b => [6, 7, 8] """ function selectRange(d::Dict{Symbol, <:AbstractVector}, range) newDict = Dict{Symbol, AbstractVector}() for (k, v) in d newDict[k] = v[range] end return newDict end """ Recursively traverses a nested dictionary structure using a vector of keys and assigns a value to the final key. Creates intermediate dictionaries if they don't exist. # Arguments - `dict::Dict` The root dictionary to traverse and modify. - `accessArray::Array{Symbol}` A vector of symbols representing the key path to traverse. - `valueToAssign` The value to assign at the final key in the path. # Return - `0` on success (value assigned) - `1` if the path cannot be traversed (missing intermediate keys) # Notes - The function walks through each key in `accessArray` except the last one, expecting intermediate keys to already exist in the dictionary. - If any intermediate key is missing, the function returns `1` without modifying the dictionary. - The final key in `accessArray` receives the `valueToAssign`. # Examples ```jldoctest julia> using GeneralUtils julia> d = Dict( :a1=> Dict(:c=> 5), :a2=> Dict( :k=> 10, :b=> Dict( :s=> "target", ) ) ) julia> assignDict!(d, [:a2, :b, :s], "wow") 0 julia> d[:a2][:b][:s] "wow" ``` """ function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign) wd = nothing for i in accessArray println(i) if i != accessArray[end] if wd === nothing && haskey(dict, i) wd = Ref(dict[i]) elseif wd.x !== nothing && haskey(wd.x, i) wd = Ref(wd.x[i]) else return 1 # error, no target key in a given dict. end else wd.x[i] = valueToAssign return 0 end end end """ Converts hour (0-23) and minute (0-59) into a Julia `Time` object using 12-hour format with AM/PM indicator. # Arguments - `h::Integer` Hour in 24-hour format (0 to 23). - `m::Integer` Minute (0 to 59). # Return - A `Time` object representing the time in 12-hour format with AM/PM. # Notes - Hours 0 and 12 are special cases: 0 becomes 12 AM, 12 becomes 12 PM. - Hours 1-11 remain the same with "am" suffix. - Hours 13-23 are converted to 1-11 with "pm" suffix. - Minutes less than 10 are zero-padded. # Examples ```jldoctest julia> using GeneralUtils julia> iTime(0, 30) 12:30 AM julia> iTime(9, 15) 9:15 AM julia> iTime(12, 0) 12:00 PM julia> iTime(14, 5) 2:05 PM ``` """ function iTime(h::Integer, m::Integer) if h == 0 h = 12 ampm = "am" elseif 1 <= h <= 11 ampm = "am" elseif h == 12 ampm = "pm" elseif 13 <= h <= 23 h = h - 12 ampm = "pm" else error("hour out of range") end m = m < 10 ? "0$m" : m t = "$h:$m$ampm" return Time(t, "HH:MMp") end """ replace a number according to the limit if value is lower than lowerbound return lowerbound replacement value if value is more than upperbound return upperbound replacement value # Example limitvalue(4, (-5 => 0), (5 => 5)) """ function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair) lwLimit, lwReplace = lowerbound upLimit, upReplace = upperbound if v < lwLimit v = lwReplace elseif v > upLimit v = upReplace else end return v end """ Assigns elements from matrix `b` to matrix `a` using the Cartesian indices of `b`. Elements are copied in the order they appear when iterating over `b`, and placed into `a` at the corresponding Cartesian positions of `b`. # Arguments - `a` Target matrix where values from `b` will be assigned. - `b` Source matrix whose Cartesian indices determine where values are placed in `a`. # Return - `nothing` # Notes - The function iterates through `b` in column-major order (Julia's default), retrieving each element's Cartesian index and assigning it to the same position in `a`. - Matrix `a` must have sufficient size to accommodate all Cartesian indices from `b`; otherwise, an `BoundsError` may occur. # Examples ```jldoctest julia> using GeneralUtils julia> a = zeros(4, 4); julia> b = [1 2; 3 4]; julia> cartesianAssign!(a, b); julia> a[1:2, 1:2] 2×2 Matrix{Float64}: 1.0 3.0 2.0 4.0 ``` """ function cartesianAssign!(a, b) for (i, v) in enumerate(b) a[CartesianIndices(b)[i].I...] = v end return nothing end function sumAlongDim3(a::Array) totalDim = length(size(a)) if totalDim == 3 d1, d2, d3 = size(a) r = zeros(1, 1, d3) for i in 1:d3 view(r, 1, 1, i) .= sum(a[:, :, i]) end elseif totalDim == 4 d1, d2, d3, d4 = size(a) r = zeros(1, 1, d3, d4) for j in 1:d4 for i in 1:d3 view(r, 1, 1, i, j) .= sum(a[:, :, i, j]) end end else error("this condition is not define yet") end return r end """ ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all slice of 3D another matrix ,b, and concatenate at the 4th dimension. Example julia> input = rand(32, 32, 128) # batch at 3rd dim julia> weight = rand(32, 32, 1024) julia> r = matMul3Dto3DmanyTo1batch(input, weight); julia> size(r) (32, 32, 1024, 128) """ function matMul3Dto3DmanyTo1batch(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) asize = [size(a)...] bsize = [size(b)...] if resultStorage === nothing resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3]) end c = [slice .* b for slice in eachslice(a, dims=3)] resultStorage .= cat(c..., dims=4) return resultStorage end """ GPU kernel """ function matMul3Dto3DmanyTo1batch_gpu!(a, b, resultStorage, linearToCartesian) i = (blockIdx().x - 1) * blockDim().x + threadIdx().x if i <= size(a, 3) # guard against unused threads to accessing memory out of bound cartesianIndex = linearToCartesian(i, size(b)) # example for how to send "inner" function to gpu # @cuprintln("gpu thread $i $cartesianIndex[2]") @. @views resultStorage[:, :, :, i] = a[ :, :, i] * b # view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b # alternative code # @view(resultStorage[:, :, :, i]) .= @view(a[ :, :, i]) .* b # alternative code end return nothing end """ ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all batch of another 4D matrix ,b, and concatenate at the 4th dimension. Example julia> julia> a = rand(2,2,3) # 3-batches julia> b = rand(2,2,4,3) # 3-batches julia> r = GeneralUtils.matMul_3Dto4D_batchwise(a, b); julia> size(r) (2, 2, 4, 3) """ function matMul_3Dto4D_batchwise(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing) if size(a, 3) != size(b, 4) error("batch number of a and b must be equal") end if resultStorage === nothing resultStorage = zeros(size(b, 1), size(b, 2), size(b, 3), size(a, 3)) end for i in 1:size(a, 3) view(resultStorage, :, :, :, i) .= a[:, :, i] .* b[:, :, :, i] end return resultStorage end """ GPU-compatible linear index to cartesian index conversion """ function linearToCartesian(i::Int, arraySize::NTuple{4,Int}) # Check that the linear coordinate is valid # prod(arraySize) is the same as *(arraySize...). they multipy all elements in an array. # but this code use prod() because splat breaks GPU performance if i < 1 || i > prod(arraySize) error("Invalid linear coordinate") end # Extract the dimensions of the matrix n1, n2, n3, n4 = arraySize # Compute the cartesian coordinate using rem and div functions i1 = ((i-1) % (n1)) + 1 # +1 convert 0-based to 1-based index i2 = ((i-1) ÷ (n1)) % n2 + 1 i3 = ((i-1) ÷ (n1*n2)) % n3 + 1 i4 = (i-1) ÷ (n1*n2*n3) + 1 # Return the cartesian coordinate as a tuple return (i1, i2, i3, i4) end """ return a vector with true at max value and false for other value. if vector is all-zeros, return all-false vector. """ function vectorMax(x) if sum(isNotEqual.(x, 0)) == 0 # guard against all-zeros array # instead of returning all-zeros original vector, # return all-false vector to prevent type instability return isNotEqual.(x, 0) else return isequal.(x, maximum(x)) end end function multiply_last(matrix, x, n) # X is the scalar to multiply # matrix is the column-major 2D matrix # n is the number of elements to be multiplied, starting from the last one # returns a new matrix with the same shape as the original one # get the number of rows and columns of the matrix rows, cols = size(matrix) # create a copy of the matrix to avoid mutating the original one result = copy(matrix) # loop over the last n elements in column-major order for i in (rows * cols - n + 1):(rows * cols) # get the row and column indices of the current element row = (i - 1) % rows + 1 col = (i - 1) ÷ rows + 1 # multiply the element by X and store it in the result matrix result[row, col] *= x end # return the result matrix return result end function multiplyRandomElements(A, x, n, rng=MersenneTwister(1234)) # rng is a random number generator object, see https://docs.julialang.org/en/v1/stdlib/Random/ # x is a scalar value to multiply by # A is a column-major 2D matrix or a vector # n is the number of elements to be multiplied # returns a new array with n randomly chosen distinct elements multiplied by x B = copy(A) # make a copy of A to avoid mutating it d = ndims(A) # get the number of dimensions of A if d == 1 # if A is a vector m = length(A) # get the length of A indices = collect(1:m) # create an array of indices from 1 to m shuffle!(rng, indices) # shuffle the indices in-place using the RNG for i in 1:n # loop n times j = indices[i] # get the i-th shuffled index B[j] *= x # multiply the element at j by x end elseif d == 2 # if A is a matrix m = size(A, 1) # number of rows in A p = size(A, 2) # number of columns in A indices = collect(1:m*p) # create an array of linear indices from 1 to m*p shuffle!(rng, indices) # shuffle the indices in-place using the RNG for i in 1:n # loop n times j = indices[i] # get the i-th shuffled index B[j] *= x # multiply the element at j by x end else # if A is neither a vector nor a matrix error("A must be a vector or a matrix") end return B # return the new array end """ Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and replace matrix A's elements of the same position with value, a. Example julia> mask = rand([-1,0,1],4,4,1) julia> A = rand(4,4,1) julia> C = replaceElements(mask, A, -1, 5.0, 3) """ function replaceElements(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number, n::Int=0; rng::AbstractRNG=MersenneTwister(1234)) """ Prompt Write a julia function to operate on column-major 3D matrix. The function randomly choose elements in matrix mask that has value markValue and replace elements in matrix A at the same position with value a. The choosing randomness is controlled by rng function. I also want to specify how many elements to be replaced. """ total_x_tobeReplced = sum(isequal.(mask, markValue)) if n == 0 || n > total_x_tobeReplced n = total_x_tobeReplced end # check if mask and A have the same size if size(mask) != size(A) error("mask and A must have the same size") end C = copy(A) # get the indices of elements in mask that equal markValue indices = findall(x -> x == markValue, mask) # shuffle the indices using the rng function shuffle!(rng, indices) # select the first n indices selected = indices[1:n] # replace the elements in A at the selected positions with a for i in selected C[i] = a end return C end """ Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and replace matrix A's elements of the same position with value, a. if n == 0, all marked value is replaced Example julia> mask = rand([-1,0,1],4,4,1) julia> A = rand(4,4,1) julia> C = replaceElements(mask, A, -1, 5.0, 3) """ function replaceElements!(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number, n::Int=0; rng::AbstractRNG=MersenneTwister(1234)) total_x_tobeReplced = sum(isequal.(mask, markValue)) remaining = 0 if n == 0 || n > total_x_tobeReplced remaining = n - total_x_tobeReplced n = total_x_tobeReplced end # check if mask and A have the same size if size(mask) != size(A) error("mask and A must have the same size, mask $(size(mask)) A $(size(A))") end # get the indices of elements in mask that equal markValue indices = findall(x -> x == markValue, mask) # shuffle the indices using the rng function shuffle!(rng, indices) # select the first n indices selected = indices[1:n] # replace the elements in A at the selected positions with a for i in selected A[i] = a end return remaining end """ Replace n elements that has value x with user specified value a. """ function replaceElements(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234)) total_x_tobeReplced = sum(isequal.(A, x)) if n == 0 || n > total_x_tobeReplced n = total_x_tobeReplced end B = copy(A) # A is a column-major 3D matrix # x is the value to be replaced # a is the new value # rng is a random number generator function # n is the number of elements to be replaced # find the indices of elements in A that equal x indices = findall(==(x), B) # shuffle the indices using the rng function shuffle!(rng, indices) # select the first n indices selected = indices[1:n] # replace the elements at the selected indices with a for i in selected B[i] = a end # return the modified matrix A return B end function replaceElements!(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234)) total_x_tobeReplced = sum(isequal.(A, x)) if n == 0 || n > total_x_tobeReplced n = total_x_tobeReplced end # A is a column-major 3D matrix # x is the value to be replaced # a is the new value # rng is a random number generator function # n is the number of elements to be replaced # find the indices of elements in A that equal x indices = findall(==(x), A) # shuffle the indices using the rng function shuffle!(rng, indices) # select the first n indices selected = indices[1:n] # replace the elements at the selected indices with a for i in selected A[i] = a end end """ Get characters between specified characters. # Arguments - `text::T` a text being searched - `startChar::Char` start character - `endChar::Char` end character # Keyword Arguments - `endCharLocation::String` end character position after startChar. Can be "next" or "end". "next" means the closed endChar just after startChar. "end" means the furthest endChar. - `includeChar::Bool` whether to include the startChar and endChar. Default is true # Return the characters between specified characters. # Example ```jldoctest julia> using Revise julia> using GeneralUtils julia> text = "{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}\n\n" julia> GeneralUtils.getStringBetweenCharacters(text, '{', '}', endCharLocation="end") "{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}" ``` """ function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char; endCharLocation::String="next", includeChar::Bool=true)::String where {T<:AbstractString} # get the position of the startChar startCharPosition = findfirst(startChar, text) endCharPosition = nothing if endCharLocation == "end" # get the first position of the endChar coming from the end of text endCharPosition = findlast(endChar, text) elseif endCharLocation == "next" # get the first position of the endChar after startCharPosition endCharPosition = findnext(endChar, text, startCharPosition + 1) else error("endCharPositio must be \"end\" or \"next\"") end @show startCharPosition, endCharPosition # get characters between startChar and endChar from text extractedText = text[startCharPosition:endCharPosition] # convert substring to string extractedText = string(extractedText) extractedText = includeChar == true ? extractedText : extractedText[2:end-1] return extractedText end """ Recursively creates nested dictionary paths if they do not exist and assigns a value to the final key. Similar to `mkpath()` but for dictionaries. # Arguments - `dict::Union{Dict{Symbol, Any}, Dict{String, Any}}` The target dictionary to traverse and modify. Must use consistent key types (either all `String` or all `Symbol`). - `addkeys::Union{Vector{String}, Vector{Symbol}}` A vector of keys representing the path to traverse. Intermediate dictionaries are created if they don't exist. - `value` The value to assign at the final key in the path. # Return - The assigned `value`. # Notes - The function ensures key type consistency: the type of keys being added must match the type of existing keys in the dictionary. - Intermediate dictionaries are automatically created with the appropriate key type when they don't exist. - The function walks through each key in `addkeys` except the last one, creating intermediate dictionaries as needed, and assigns `value` to the final key. # Examples ```jldoctest julia> using GeneralUtils julia> d = Dict("a" => Dict("b" => 10)) julia> mkDictPath!(d, ["a", "v", "x", "y", "z"], 42) 42 julia> d["a"]["v"]["x"]["y"]["z"] 42 ``` """ function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}}, addkeys::Union{Vector{String}, Vector{Symbol}}, value) # new key and existing key must be the same type if !isempty(dict) existingKeys = [key for key in keys(dict)] if typeof(existingKeys[1]) != typeof(addkeys[1]) error("Type of keys being added is $(typeof(addkeys[1])) but type of existing keys is $(typeof(existingKeys[1]))") end end for key in addkeys[1:end-1] if !haskey(dict, key) key_type = eltype(keys(dict)) dict[key] = Dict{key_type, Any}() end dict = dict[key] end return dict[addkeys[end]] = value end """ Retrieves a value from a nested dictionary by traversing a vector of keys. Creates intermediate dictionaries if they don't exist. # Arguments - `dict::Dict` The root dictionary to traverse. - `keys::Vector` A vector of keys representing the path to traverse. Each key in the vector is used to access the next level of nesting. # Return - The value at the final key in the path. # Notes - Errors with `ArgumentError` if any intermediate key is missing from the dictionary path. - The function walks through each key in `keys` except the last one, expecting intermediate keys to exist in the dictionary. - The final key in `keys` is used to retrieve the value. # Examples ```jldoctest julia> using GeneralUtils julia> d = Dict(:a => Dict(:b => 10)) julia> getDictPath(d, [:a, :b]) 10 ``` """ function getDictPath(dict::Dict, keys::Vector) current_dict = dict for key in keys[1:end-1] if haskey(current_dict, key) current_dict = current_dict[key] else throw(ArgumentError("Key $key not found in dictionary")) end end last_key = keys[end] if haskey(current_dict, last_key) return current_dict[last_key] else throw(ArgumentError("Key $last_key not found in dictionary")) end end """ detectKeywordVariation(keywords::AbstractVector{String}, text::String) -> Dict{String, Union{Array, Nothing}} Detects and collects all case-variant occurrences of multiple keywords in the text. This function processes each keyword individually and returns an array of matched variations for each keyword. # Arguments - `keywords::AbstractVector{String}` Vector of keywords to search for - `text::String` The text to search in # Returns - `Dict{String, Array}` Returns a dictionary mapping each keyword to an array of matched variations found in the text # Examples ```jldoctest julia> detectKeywordVariation(["test", "example", "cat"], "This is a Test EXAMPLE") Dict{String, Array}("test" => ["Test"], "example" => ["EXAMPLE"], "cat" => nothing) """ function detectKeywordVariation(keywords::T, text::String)::Dict{String, Union{Array, Nothing}} where {T<:AbstractVector} kw = Dict{String, Union{Array, Nothing}}() # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list for keyword in keywords ws = detectKeywordVariation.(keyword, text) total = sum(issomething.(ws)) if total != 0 kw[keyword] = ws else kw[keyword] = nothing end end return kw end """ detectKeywordVariation(keyword::String, text::String) -> Union{Nothing, Array{String}} Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase). # Arguments: - `keyword::String` The keyword to search for - `text::String` The text to search in # Returns: - `Union{Nothing, Array{String}}` Returns an array of matched keyword variations if found, otherwise returns nothing # Examples: ```jldoctest julia> detectKeywordVariation("test", "This is a Test case") ["Test"] julia> detectKeywordVariation("error", "NO ERRORS FOUND") ["ERRORS"] julia> detectKeywordVariation("missing", "complete data") nothing ``` """ function detectKeywordVariation(keyword::String, text::String)::Union{Nothing, Array{String}} # Define the keyword variations to search for wordVariations = [uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)] # wordVariations may duplicate keyword keyword_variations = [keyword] for i in wordVariations i != keyword ? push!(keyword_variations, i) : nothing end _splittext = string.(strip.(split(text, " "))) splittext = String[] # remove . after a word for i in _splittext if length(i) != 0 && i[end] ∈ ['.'] word = string(i[1:end-1]) push!(splittext, word) else push!(splittext, i) end end result = String[] for variation in keyword_variations # if length of both word is equals then it is a whole word otherwise it is part of part of other word r = findIndex(splittext, variation) if isempty(r[2]) # skip else # if variation > 1 add them all so this function detect duplicate keyword variations = [variation for i in eachindex(r[2])] result = vcat(result, variations) end end return result end """ Convert text into a dictionary with a given keywords. This function use keywords to slice a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text. The left most string which has no keyword will be discarded. WARNING, ordering is important # Arguments - `text::String` A text to be converted. - `keywords::Vector{String}` A list of keywords to be used to slice the text. These keywords also be the resulting dict keys. # Keyword Arguments - `rightmarker::String` A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", the function will search for "plan:" otherwise the function will search for "plan". The marker will not be in the resulting dict keys. - `symbolkey::Bool` If true, resulting dict's key will be Symbols, otherwise string. - `lowercasekey::Bool` set resulting dict's key to be lowercase # Return - `d::OrderedDict` # Example ```jldoctest julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep" julia> sample_keywords = ["thought", "plan", "action"] julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true) julia> println(resultdict) OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", :plan => "wake up and going out", :action => "1. wake up 2. eat 3. sleep") ``` """ function textToDict(text::String, detectKeywords::Vector{String}; dictKey::Union{Vector{String}, Nothing}=nothing, symbolkey::Bool=false, lowercasekey::Bool=false )::OrderedDict # make sure this function detect variation of a work e.g. agent, Agent, AGENT kw = [] # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list for keyword in detectKeywords detected = detectKeywordVariation(keyword, text) if detected !== nothing push!(kw, detected) else error("Keyword $keyword not found in text: $text") end end if typeof(kw[1]) <: AbstractArray kw = reduce(vcat, kw) end od1, od2 = if symbolkey OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}() else OrderedDict{String, Any}(), OrderedDict{String, Any}() end remainingtext = text dictKey_ = reverse(dictKey) # process text from back to front rkw = reverse(kw) for (i,keyword) in enumerate(rkw) # Find the position of the keyword in the text keywordidx = findlast(keyword, remainingtext) dKey = dictKey_[i] if keywordidx !== nothing substr = remainingtext[keywordidx[end]+1:end] str = string(strip(substr)) # Removes both leading and trailing whitespace. _key = lowercasekey == true ? lowercase(dKey) : dKey key = symbolkey == true ? Symbol(_key) : _key od1[key] = str remainingtext = remainingtext[1:keywordidx[1]-1] else error("""keyword "$keyword" not found in the provided text: $text """) end end # correct the order ks = reverse([i for i in keys(od1)]) for k in ks k = symbolkey == true ? Symbol(k) : k od2[k] = od1[k] end return od2 end """ Recursively convert dictionary into an HTML string representation. The function walks a nested `AbstractDict` structure and produces a well-formed HTML string where each dictionary key becomes an HTML tag. Nested dictionaries become nested tags, and scalar values (numbers, strings, etc.) become the text content of leaf tags. # Arguments - `d::AbstractDict` The dictionary to convert. Keys must be strings or symbols that form valid HTML tag names. # Keyword Arguments - `indent_level::Integer=1` Initial indentation level for the output. Each recursive level increases indentation by one. - `indent_str::String=" "` String used for each indentation level. # Return - A single HTML string representing the dictionary structure, with proper opening and closing tags and appropriate indentation. # Notes - Keys are sorted alphabetically for deterministic output. - Works recursively: dictionary values produce nested tags; non-dictionary values are placed as text between opening/closing tags. # Examples ```jldoctest julia> d = Dict( "html" => Dict( "head" => Dict("title" => "Test"), "body" => Dict( "h1" => "Hello", "p" => "World" ) ) ); julia> println(dict_to_string_html(d))
World
``` """ function dict_to_string_html(d::AbstractDict; indent_level=1, indent_str=" ") lines = String[] padding = indent_str ^ indent_level # Sort keys for predictable, clean output for k in sort(collect(keys(d))) v = d[k] if v isa AbstractDict # Open tag, recurse for children, then close tag push!(lines, "$padding<$k>") ind_level = indent_level + 1 push!(lines, dict_to_string_html(v; indent_level=ind_level, indent_str=indent_str)) push!(lines, "$padding$k>") else # Leaf node: put key and value on a single line push!(lines, "$padding<$k>$v$k>") end end return join(lines, "\n") end end # module