1806 lines
54 KiB
Julia
1806 lines
54 KiB
Julia
module interface
|
||
|
||
|
||
export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue, replaceMoreThan,
|
||
replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul3Dto3DmanyTo1batch,
|
||
matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax,
|
||
multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween,
|
||
isLess, allTrue, getStringBetweenCharacters, mkDictPath!, dict_to_string_html,
|
||
getDictPath, detectKeywordVariation, textToDict, dictify, ordereddictify
|
||
|
||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV
|
||
using ..util, ..communication
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
noNegative!(a::AbstractVector) = replace!(x -> x < 0 ? 0 : x, a)
|
||
findNotZero(x::AbstractVector) = findall( (!iszero).(x) )
|
||
replaceMoreThan(x, target, replaceValue) = x > target ? replaceValue : x
|
||
replaceMoreThan(x, target, a, b) = x > target ? a : b
|
||
replaceLessThan(x, target, replaceValue) = x < target ? replaceValue : x
|
||
replaceLessThan(x, target, a, b) = x < target ? a : b
|
||
replaceBetween(x, lowerbound, upperbound, replaceValue) = lowerbound < x < upperbound ? replaceValue : x
|
||
precision(x::Array{<:Array}) = ( std(mean.(x)) / mean(mean.(x)) ) * 100
|
||
precision(x::Array) = std(x) / mean(x) * 100
|
||
replaceAt!(x::AbstractVector, ind::Number, value::Number) = x[ind] = value
|
||
notZero(x::AbstractVector) = (!iszero).(x)
|
||
Zero(x::AbstractVector) = iszero.(x)
|
||
isNan(x::AbstractVector) = isnan.(x)
|
||
isInf(x::Number) = abs(x) === Inf
|
||
isInf(x::AbstractVector) = isinf.(x)
|
||
isNotEqual(x::Number, target::Number) = isequal(isequal(x, target), 0)
|
||
isBetween(x, lowerlimit, upperlimit) = lowerlimit < x < upperlimit ? true : false
|
||
absolute(x::AbstractVector) = abs.(x)
|
||
vecEleMul(x::AbstractVector, y::AbstractVector) = x .* y
|
||
vecEleMul(x::Number, y::AbstractVector) = x .* y
|
||
expDecay(initialValue::Number, decayFactor::Number, timePass::Number) =
|
||
initialValue * (1 - decayFactor)^timePass
|
||
mul!(x::AbstractVector, y::AbstractVector) = x .*= y
|
||
mul(x::AbstractVector, y::AbstractVector) = x .* y
|
||
allTrue(args...) = false ∈ [args...] ? false : true
|
||
|
||
ReLu(x::Number) = max(0, x)
|
||
|
||
updateVector!(x::AbstractVector, target::Number) = x .= target
|
||
updateVector!(x::AbstractVector, target::AbstractArray) = x .= target
|
||
|
||
function selectAdd!(x::AbstractVector, ind::AbstractVector, value::AbstractVector)
|
||
@. x = x + (ind * value)
|
||
end
|
||
|
||
|
||
|
||
""" FindIndex(input::String, target::Char)
|
||
|
||
Arguments:
|
||
text, input text
|
||
target, target character
|
||
Return:
|
||
(a bool vector of match/not match, position vector of the matched)
|
||
|
||
Example:
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> text = "Hello World!"
|
||
julia> findIndex(text, 'l')
|
||
(Bool[0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0], [3, 4, 10])
|
||
```
|
||
"""
|
||
function findIndex(text::String, target::Char)
|
||
charlist = [i for i in text]
|
||
match_position = isequal.(charlist, target)
|
||
match_index = findall(isequal.(match_position, 1))
|
||
|
||
return match_position, match_index
|
||
end
|
||
|
||
function findIndex(input::Array, target::Number)
|
||
match_position = isequal.(input, target)
|
||
match_index = findall(match_position)
|
||
return match_position, match_index
|
||
end
|
||
|
||
# function findIndex(input::Array, target::Array)
|
||
# match_position = isone.(zeros(length(input)))
|
||
# for i in target
|
||
# match_position = match_position + isequal.(input, i)
|
||
# end
|
||
|
||
# match_position = replaceMoreThan.(match_position, 1)
|
||
# match_index = findall(isone.(match_position)) # Findall donot work with Int64 vector [1, 0, 0, 1].
|
||
# # It only works with BitVector. isone() converts Int64 vector [1, 0, 0, 1] into
|
||
# # BitVector [1, 0, 0, 1]
|
||
|
||
# return match_position, match_index
|
||
# end
|
||
|
||
function findIndex(input::Array, target::Symbol)
|
||
match_position = isequal.(input, target)
|
||
match_index = findall(match_position)
|
||
|
||
return match_position, match_index
|
||
end
|
||
|
||
function findIndex(collection::Array{String}, target::String)
|
||
match_position = isequal.(collection, target)
|
||
match_index = findall(match_position)
|
||
|
||
return match_position, match_index
|
||
end
|
||
|
||
function findIndex(collection::Array{String}, target::Array{String})
|
||
match_position = nothing
|
||
match_index = nothing
|
||
|
||
for i in target
|
||
match_pos = isequal.(collection, i)
|
||
match_ind = findall(match_pos)
|
||
|
||
if match_position === nothing
|
||
match_position = match_pos
|
||
|
||
else
|
||
match_position = hcat(match_position, match_pos)
|
||
end
|
||
|
||
if match_index === nothing
|
||
match_index = match_ind
|
||
else
|
||
match_index = hcat(match_index, match_ind)
|
||
end
|
||
end
|
||
|
||
return match_position, match_index
|
||
end
|
||
|
||
function findIndex(collection::OrderedDict, target::Symbol)
|
||
collection_keys = keys(collection)
|
||
collection_keys_array = [i for i in collection_keys]
|
||
match_position = isequal.(collection_keys_array, target)
|
||
match_index = findall(match_position)
|
||
|
||
return match_position, match_index
|
||
end
|
||
|
||
function findMax(collection::AbstractVector)
|
||
maxValue, maxIndex = findmax(collection)
|
||
matchPosition = isequal.(collection, maxValue)
|
||
return maxValue, maxIndex, matchPosition
|
||
end
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
""" Reads the x-th text file from a folder, where files are listed by the OS
|
||
without explicit sorting. Returns the file number, filename, and content.
|
||
|
||
# Arguments
|
||
- `folder_path::String`
|
||
Path to the folder containing text files.
|
||
- `read_file_number::Integer=1`
|
||
Which file to read (1-based index). Defaults to the first file.
|
||
|
||
# Return
|
||
- A tuple of `(file_number::Integer, filename::String, content)` where:
|
||
- `file_number` is the index of the file that was read
|
||
- `filename` is the actual filename string
|
||
- `content` is a vector of lines from the file (or `nothing` if error)
|
||
|
||
# Notes
|
||
- Errors if `folder_path` is not a valid directory.
|
||
- Errors if `read_file_number` exceeds the number of files in the folder.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> result = read_textfile_by_index("/path/to/folder", 2)
|
||
(2, "sample.txt", ["line1", "line2", ...])
|
||
```
|
||
"""
|
||
function read_textfile_by_index(folder_path::String, read_file_number::Integer=1)
|
||
if isdir(folder_path)
|
||
filenumber = length(readdir(folder_path))
|
||
|
||
if read_file_number > filenumber
|
||
error("you specified read_file_number = $read_file_number which is out
|
||
of range, the cleaned data folder has only $filenumber files")
|
||
return nothing, nothing, nothing
|
||
else
|
||
content = 0
|
||
# open each file in the directory and read
|
||
filename = readdir(folder_path, join=true, sort=false)[read_file_number]
|
||
f = open(filename)
|
||
content = readlines(f)
|
||
# content = read(f)
|
||
close(f)
|
||
end
|
||
|
||
return read_file_number, filename, content
|
||
else
|
||
error("ERROR no file or folder at $folder_path")
|
||
return nothing, nothing, nothing
|
||
end
|
||
|
||
end
|
||
|
||
|
||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into an OrderedDict.
|
||
|
||
The function walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`,
|
||
`Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where
|
||
every dictionary-like node is an `OrderedDict` and every array-like node is a `Vector{Any}`.
|
||
Scalar values (numbers, strings, booleans, `nothing`, etc.) are returned unchanged.
|
||
Does **not** mutate the input; it always allocates new containers.
|
||
|
||
# Arguments
|
||
- `x`
|
||
Any Julia value. If `x` is an `AbstractDict` it will be converted to an `OrderedDict`;
|
||
if it is an `AbstractArray` its elements will be processed recursively.
|
||
|
||
# Keyword Arguments
|
||
- `keytype::Type=Any`
|
||
The key type for the output OrderedDict. Use `String` for `OrderedDict{String,Any}`,
|
||
`Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types.
|
||
- `sort_order::Union{Nothing, Vector}=nothing`
|
||
Vector of keys specifying the desired order. Keys are arranged in the specified order
|
||
first, followed by any remaining keys.
|
||
|
||
# Return
|
||
- A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and `Vector{Any}`
|
||
that mirrors the input shape but uses ordered Julia containers.
|
||
|
||
# Notes
|
||
- The function treats any `AbstractDict` as a mapping source, so it works with
|
||
`JSON.Object`, `Dict`, `OrderedDict`, etc.
|
||
- Arrays are returned as `Vector{Any}` with their elements processed recursively.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using JSON, DataStructures
|
||
julia> d = Dict(
|
||
"a" => 4,
|
||
"b" => 6,
|
||
"c" => Dict(
|
||
"d"=>7,
|
||
:e=>Dict(
|
||
"f"=>"hey",
|
||
"g"=>Dict(
|
||
"world"=>[1, "2", 3, Dict(:dd=>4.7)]
|
||
)
|
||
)
|
||
)
|
||
)
|
||
|
||
julia> jsonstring = JSON.json(d)
|
||
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
||
julia> A2 = dictify(A1; keytype=String)
|
||
OrderedDict{String,Any} with 3 entries:
|
||
"a" => 4
|
||
"b" => 6
|
||
"c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||
|
||
julia> A3 = dictify(A1; keytype=Symbol)
|
||
OrderedDict{Symbol,Any} with 3 entries:
|
||
:a => 4
|
||
:b => 6
|
||
:c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||
|
||
julia> B1 = dictify(d; keytype=String)
|
||
OrderedDict{String, Any} with 3 entries:
|
||
```
|
||
|
||
**With sort_order:**
|
||
```jldoctest
|
||
julia> d = Dict("a"=>1, "b"=>2, "c"=>3)
|
||
julia> dictify(d; sort_order=["c", "a"])
|
||
OrderedDict{String,Int} with 3 entries:
|
||
"c" => 3
|
||
"a" => 1
|
||
"b" => 2
|
||
```
|
||
"""
|
||
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||
)::OrderedDict where {T<:AbstractDict}
|
||
# Dict-like objects
|
||
out = OrderedDict{keytype, Any}()
|
||
|
||
# 1. Process and normalize all keys from the input dictionary
|
||
processed_dict = OrderedDict{keytype, Any}()
|
||
for (k, v) in x
|
||
if keytype === String
|
||
newk = string(k)
|
||
elseif keytype === Symbol
|
||
newk = Symbol(string(k))
|
||
else
|
||
newk = k
|
||
end
|
||
processed_dict[newk] = dictify(v; keytype=keytype, sort_order=sort_order)
|
||
end
|
||
|
||
# 2. If a sort order is specified, apply it
|
||
if !isnothing(sort_order)
|
||
# Normalize the sort_order elements to match the requested keytype
|
||
normalized_order = map(sort_order) do tk
|
||
if keytype === String
|
||
return string(tk)
|
||
elseif keytype === Symbol
|
||
return Symbol(string(tk))
|
||
else
|
||
return tk
|
||
end
|
||
end
|
||
|
||
# First, insert keys that match the requested order
|
||
for target_key in normalized_order
|
||
if haskey(processed_dict, target_key)
|
||
out[target_key] = processed_dict[target_key]
|
||
end
|
||
end
|
||
|
||
# Then, append any remaining keys that weren't in the sort_order
|
||
for (k, v) in processed_dict
|
||
if !haskey(out, k)
|
||
out[k] = v
|
||
end
|
||
end
|
||
else
|
||
# If no sort order is given, just use the processed dict
|
||
out = processed_dict
|
||
end
|
||
|
||
return out
|
||
end
|
||
|
||
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||
) where {T<:AbstractArray}
|
||
return [dictify(element; keytype=keytype, sort_order=sort_order) for element in x]
|
||
end
|
||
|
||
function dictify(x; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||
)
|
||
return x
|
||
end
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary.
|
||
The function walks any nested structure composed of AbstractDict (e.g., JSON.Object,
|
||
Dict, OrderedDict) and AbstractArray and produces a new tree where
|
||
every dictionary-like node is an OrderedDict{Any,Any} and every array-like
|
||
node is a Vector{Any}. Scalar values (numbers, strings, booleans,
|
||
nothing, etc.) are returned unchanged.
|
||
Does **not** mutate the input; it always allocates new containers.
|
||
|
||
# Arguments
|
||
- `x`
|
||
Any Julia value. If x is an AbstractDict it will be converted to an
|
||
OrderedDict{Any,Any}. if it is an AbstractArray its elements will be
|
||
processed recursively.
|
||
|
||
# Keyword Arguments
|
||
- `keytype::Type=Any`
|
||
The key type for the output Dict. Use `String` for `OrderedDict{String,Any}`, `Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types.
|
||
|
||
# Return
|
||
- A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and
|
||
`Vector{Any}` that mirrors the input shape but uses ordered Julia containers.
|
||
|
||
# Notes
|
||
- The function treats any `AbstractDict` as a mapping source, so it works with
|
||
`JSON.Object`, `Dict`, `OrderedDict`, etc.
|
||
- Arrays are returned as `Vector{Any}` with their elements processed recursively.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using JSON
|
||
julia> d = Dict(
|
||
"a" => 4,
|
||
"b" => 6,
|
||
"c" => Dict(
|
||
"d"=>7,
|
||
:e=>Dict(
|
||
"f"=>"hey",
|
||
"g"=>Dict(
|
||
"world"=>[1, "2", 3, Dict(:dd=>4.7)]
|
||
)
|
||
)
|
||
)
|
||
)
|
||
|
||
julia jsonstring = JSON.json(d)
|
||
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
||
julia> A2 = ordereddictify(A1; keytype=String)
|
||
OrderedDict{String,Any} with 3 entries:
|
||
"a" => 4
|
||
"b" => 6
|
||
"c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||
|
||
julia> A3 = ordereddictify(A1; keytype=Symbol)
|
||
OrderedDict{Symbol,Any} with 3 entries:
|
||
:a => 4
|
||
:b => 6
|
||
:c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||
|
||
julia> B1 = ordereddictify(d; keytype=String)
|
||
OrderedDict{String, Any} with 3 entries:
|
||
"c" => OrderedDict{String, Any}("e"=>OrderedDict{String, Any}("f"=>"hey", "g"=>OrderedDict{String, Any}("world"=>Any[1, "2", 3, OrderedDict{String, Any}("dd"=>4.7)])), "d"=>7)
|
||
"b" => 6
|
||
"a" => 4
|
||
```
|
||
Ref. https://github.com/andyferris/Dictionaries.jl
|
||
"""
|
||
function ordereddictify(x; keytype::Type=Any)
|
||
# Dict-like objects
|
||
if x isa AbstractDict
|
||
# choose output key type container
|
||
out = OrderedDict{keytype,Any}()
|
||
for (k,v) in x
|
||
if keytype === String
|
||
newk = string(k)
|
||
elseif keytype === Symbol
|
||
newk = Symbol(string(k))
|
||
else
|
||
newk = k
|
||
end
|
||
out[newk] = ordereddictify(v; keytype=keytype)
|
||
end
|
||
return out
|
||
# Arrays / vectors: map elements recursively and return a Vector{Any}
|
||
elseif x isa AbstractArray
|
||
return [ordereddictify(element; keytype=keytype) for element in x]
|
||
# everything else: return as-is (primitives, numbers, strings, etc.)
|
||
else
|
||
return x
|
||
end
|
||
end
|
||
|
||
#----------------------------------------------100---------------------------------------------
|
||
|
||
"""
|
||
print time of cpu executtion at the line inwhich this macro is used
|
||
"""
|
||
macro timeline(expr)
|
||
quote
|
||
print("line ", $(__source__.line), ": ")
|
||
@time $(esc(expr))
|
||
end
|
||
end
|
||
|
||
|
||
batchindex(batch_counter::Number, batch_size::Number; offset=0) =
|
||
(offset + (batch_counter-1) * batch_size + 1) : offset + (batch_counter * batch_size)
|
||
|
||
function flip_true_false(x::Bool)
|
||
if x == true
|
||
x = false
|
||
elseif x == false
|
||
x = true
|
||
else
|
||
error("undefined condition line $(@__LINE__)")
|
||
end
|
||
|
||
return x
|
||
end
|
||
|
||
function flip_true_false(x::Int)
|
||
if x == 1
|
||
x = 0
|
||
elseif x == 0
|
||
x = 1
|
||
else
|
||
throw("not define input of type $(typeof(x)) yet")
|
||
end
|
||
|
||
return x
|
||
end
|
||
|
||
"""
|
||
Return drawed index
|
||
# Example
|
||
drawed_index = randomWithProb([0.5, 0.2, 0.3])
|
||
|
||
"""
|
||
randomWithProb(probability::AbstractVector) = rand(Distributions.Categorical(probability)) # return drawed index
|
||
|
||
"""
|
||
Draw from choices according to its probability.
|
||
Probability range is 0.0 to 1.0 and all probability must summed up to 1
|
||
(may get probability from NNlib's softmax function)
|
||
|
||
# Example
|
||
|
||
draw = randomChoiceWithProb([true, false, nothing], [0.5, 0.2, 0.3])
|
||
"""
|
||
function randomChoiceWithProb(choices::Array, probability::Array)
|
||
if length(choices) != length(probability)
|
||
error("random is not possible, choices array length != probability array length")
|
||
elseif sum(probability) != 1.0
|
||
error("probability does not sum to 1.0")
|
||
end
|
||
|
||
return choices[randomWithProb(probability)]
|
||
end
|
||
|
||
function randomChoiceOnTarget(target::Number, targetMatch::Number, choices::AbstractVector,
|
||
probability::AbstractVector)
|
||
if length(choices) != length(probability)
|
||
throw("random is not possible, choices array length != probability array length")
|
||
end
|
||
return target == targetMatch ? randomChoiceWithProb(choices, probability) : target
|
||
# dist = Distributions.Categorical(probability)
|
||
# draw_result = choices[rand(dist)]
|
||
end
|
||
|
||
function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector,
|
||
probability::AbstractVector)
|
||
return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,))
|
||
end
|
||
|
||
""" Compute the linearly weighted average of an array.
|
||
The function assigns weights proportional to position indices (1, 2, 3, ...) to array
|
||
elements and returns the weighted average.
|
||
|
||
# Arguments
|
||
- `a::Array`
|
||
Array of numeric values. Elements must support multiplication with numbers
|
||
and summation.
|
||
|
||
# Return
|
||
- The linearly weighted average as a floating-point number.
|
||
|
||
# Formula
|
||
For an array `a` with `n` elements, computes:
|
||
```
|
||
sum(i * a[i]) / sum(a) for i = 1 to n
|
||
```
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> a = [10, 20, 30]
|
||
julia> linearly_weighted_avg(a)
|
||
23.333333333333332
|
||
```
|
||
"""
|
||
function linearly_weighted_avg(a::Array)
|
||
total = 0.0
|
||
for (i, v) in enumerate(a)
|
||
total = total + (i * v)
|
||
end
|
||
|
||
return total / sum(a)
|
||
end
|
||
|
||
|
||
""" Convert a variable's value (String) into a Symbol.
|
||
The function takes a variable containing a String value and converts it to a Symbol
|
||
using Julia's expression interpolation mechanism.
|
||
|
||
# Arguments
|
||
- `variable`
|
||
Any variable whose value is a String. The function uses `string(variable)`
|
||
internally to obtain the value.
|
||
|
||
# Return
|
||
- A `Symbol` constructed from the string value of the input variable.
|
||
|
||
# Notes
|
||
- This function uses variable interpolation to capture the variable's
|
||
value as a Symbol. It works with any variable type that can be converted
|
||
to String.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> x = "hello"
|
||
julia> variable_str_to_symbol(x)
|
||
:hello
|
||
|
||
julia> y = "world_test"
|
||
julia> variable_str_to_symbol(y)
|
||
:world_test
|
||
```
|
||
"""
|
||
function variable_str_to_symbol(variable)
|
||
semi = :($variable)
|
||
symbol = Symbol(semi)
|
||
|
||
return symbol
|
||
end
|
||
|
||
|
||
""" get useable type of specified fieldname inside a composite struct
|
||
|
||
# Example
|
||
julia> @Base.kwdef mutable struct some_struct
|
||
a::Union{Bool, Nothing} = nothing
|
||
b::Union{Float64, Nothing} = nothing
|
||
c::Union{Int64, AbstractFloat} = 3.5
|
||
d::Union{String, Nothing} = nothing
|
||
end
|
||
|
||
julia> a = some_struct()
|
||
julia> fieldname_useable_type(some_struct, :c) =result=> [Int64, Float64]
|
||
"""
|
||
function fieldname_useable_type(somestruct, fieldname::Symbol;
|
||
test_types=[2.0, 2, true, :h, "str", 'c', missing, nothing])::Vector{DataType}
|
||
new_instance = somestruct()
|
||
useable_type = []
|
||
|
||
for i in test_types
|
||
try
|
||
new_instance.:($fieldname) = i
|
||
type = typeof(new_instance.:($fieldname))
|
||
if type ∉ useable_type
|
||
push!(useable_type, type)
|
||
end
|
||
catch
|
||
|
||
end
|
||
end
|
||
|
||
return useable_type
|
||
end
|
||
|
||
|
||
""" Draw unique elements from a list without replacement.
|
||
The function randomly selects a specified number of distinct elements from a collection,
|
||
optionally excluding certain elements from consideration. Uses in-place
|
||
shuffling for efficiency.
|
||
|
||
# Arguments
|
||
- `drawOptions::Array`
|
||
Collection of elements to draw from.
|
||
- `draw_number::Integer`
|
||
Number of unique elements to draw.
|
||
|
||
# Keyword Arguments
|
||
- `exclude_list::Union{AbstractArray,Nothing}=nothing`
|
||
Elements to exclude from the drawing pool. If `nothing`, no elements are
|
||
excluded.
|
||
|
||
# Return
|
||
- An array of `draw_number` unique elements drawn from `drawOptions`, excluding
|
||
any elements in `exclude_list`.
|
||
|
||
# Notes
|
||
- The function copies `drawOptions` and shuffles in-place, then pops elements
|
||
sequentially to ensure uniqueness.
|
||
- Errors if `draw_number` exceeds the number of available elements after
|
||
exclusion.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> options = [1, 2, 3, 4, 5]
|
||
julia> randomNoRepeat(options, 3)
|
||
[3, 1, 5]
|
||
|
||
julia> randomNoRepeat(options, 2; exclude_list=[1, 5])
|
||
[4, 2]
|
||
```
|
||
"""
|
||
function randomNoRepeat(drawOptions::Array, draw_number::Integer;
|
||
exclude_list::Union{AbstractArray,Nothing}=nothing)
|
||
draw_option = copy(drawOptions)
|
||
draw_option = isnothing(exclude_list) ? draw_option :
|
||
filter!(x -> x ∉ exclude_list, draw_option)
|
||
shuffle!(draw_option)
|
||
drawed_items = []
|
||
while length(drawed_items) < draw_number
|
||
push!(drawed_items, pop!(draw_option))
|
||
end
|
||
return drawed_items
|
||
end
|
||
|
||
""" using cron to schedule backup job by
|
||
1. sudo nano /etc/crontab <<< this is a system-wide cron file
|
||
2. to execute julia file @ 2.00am everyday add the following line at the buttom of the file
|
||
0 2 * * * root julia-1.7 /home/syncthing_backup_script.jl
|
||
|
||
Requirements using Dates
|
||
"""
|
||
function folderBackup(sourceFolderAbsolutePath::String, # absolute path to folder to be backuped
|
||
backupFolderAbsolutePath::String; # absolute path to folder used to store backup file
|
||
totalBackupFiles::Integer=7, # total backup file, the oldest will be deleted
|
||
containerName::Union{Array{String}, Nothing}=nothing) # container using source_folder
|
||
|
||
sep = (Sys.iswindows() ? "\\" : '/')
|
||
|
||
if sourceFolderAbsolutePath[end] == sep
|
||
sourceFolderAbsolutePath = sourceFolderAbsolutePath[1:end-1]
|
||
end
|
||
if backupFolderAbsolutePath[end] != sl
|
||
backupFolderAbsolutePath = backupFolderAbsolutePath * sep
|
||
end
|
||
|
||
if isdir(backupFolderAbsolutePath)
|
||
else
|
||
mkpath(backupFolderAbsolutePath)
|
||
end
|
||
|
||
# stop running docker container service
|
||
if containerName !== nothing
|
||
println("stop running services")
|
||
for i in containerName
|
||
try run(`docker stop $i`) catch; end
|
||
sleep(10) # wait for services to stop
|
||
end
|
||
end
|
||
|
||
# do backup
|
||
println("doing backup now")
|
||
timestamp = string(Dates.now())
|
||
name = split(sourceFolderAbsolutePath, sep)[end] * "--"
|
||
filename = name * timestamp * ".zip" # resulting compressed filename
|
||
run(`chmod -R a+rwx $sourceFolderAbsolutePath`)
|
||
# zip -r [destination+filename] [source folder to be zipped]
|
||
run(`zip -r $(backupFolderAbsolutePath * filename) $sourceFolderAbsolutePath`)
|
||
|
||
# check if total backup file is more than user specified, if yes, delete the oldest backup
|
||
backupFiles = readdir(backupFolderAbsolutePath)
|
||
|
||
while length(backupFiles) > totalBackupFiles
|
||
run(`rm $(backupFolderAbsolutePath * backupFiles[1])`)
|
||
backupFiles = readdir(backupFolderAbsolutePath)
|
||
end
|
||
|
||
# start docker services
|
||
if containerName !== nothing
|
||
println("start services")
|
||
for i in containerName
|
||
try run(`docker start $i`) catch; end
|
||
sleep(10) # wait for services to stop
|
||
end
|
||
end
|
||
end
|
||
|
||
function lowerclip!(data::AbstractVector, lowerbound::Number)
|
||
replace!(x -> x < lowerbound ? lowerbound : x, data)
|
||
end
|
||
|
||
function upperclip!(data::AbstractVector, upperbound::Number)
|
||
replace!(x -> x > upperbound ? upperbound : x, data)
|
||
end
|
||
|
||
function normalise(x::AbstractArray, mu, std)
|
||
ϵ = oftype(x[1], 1e-5)
|
||
μ = mu
|
||
# σ = std(x, dims=dims, mean=μ, corrected=false) # use this when Zygote#478 gets merged
|
||
σ = std
|
||
return (x .- μ) ./ (σ .+ ϵ)
|
||
end
|
||
|
||
function minMaxScaler(x::AbstractVector)
|
||
min = findmin(x)[1]
|
||
max = findmax(x)[1]
|
||
|
||
scaler(a::Number, min::Number, max::Number) = (a-min) / (max-min)
|
||
return scaler.(x, min, max)
|
||
end
|
||
|
||
""" a = [-1e200, -1e-200, 1e200, 1e-200] \n
|
||
result = vtclamp.(a, 1e-6, 1e6, -1e6, -1e-6)
|
||
"""
|
||
function customclamp(x::Number, poslo::Number, poshi::Number,
|
||
neglo::Number, neghi::Number)
|
||
signx = sign(x)
|
||
if signx == -1
|
||
if neghi < x < 0
|
||
return neghi
|
||
elseif x < neglo
|
||
return neglo
|
||
else
|
||
return x
|
||
end
|
||
elseif signx == +1
|
||
if poshi < x
|
||
return poshi
|
||
elseif 0 < x < poslo
|
||
return poslo
|
||
else
|
||
return x
|
||
end
|
||
end
|
||
end
|
||
|
||
function unitVec(x::AbstractVector)
|
||
y = √(sum(x.^2))
|
||
return x./y
|
||
end
|
||
|
||
function replaceAt!(x::AbstractVector, ind::AbstractVector, value::Number)
|
||
for i in ind
|
||
x[i] = value
|
||
end
|
||
end
|
||
|
||
function signbitVec(x::AbstractVector)
|
||
sign = signbit.(x) * 1
|
||
signVec = replace(s -> s == 0 ? -1 : s, sign)
|
||
return signVec
|
||
end
|
||
|
||
function deleteall!(x::AbstractVector)
|
||
for i in 1:length(x)
|
||
deleteat!(x, 1)
|
||
end
|
||
end
|
||
|
||
""" Select specific range of vectors in a dict, return a new dict
|
||
# Example
|
||
dict = Dict(:a => [1:5...],
|
||
:b => [6:10...])
|
||
|
||
call -> selectRange(dict, 1:3)
|
||
return -> Dict{Any, Any} with 2 entries:
|
||
:a => [1, 2, 3]
|
||
:b => [6, 7, 8]
|
||
"""
|
||
function selectRange(d::Dict{Symbol, <:AbstractVector}, range)
|
||
newDict = Dict{Symbol, AbstractVector}()
|
||
for (k, v) in d
|
||
newDict[k] = v[range]
|
||
end
|
||
|
||
return newDict
|
||
end
|
||
|
||
""" Recursively traverses a nested dictionary structure using a vector of keys
|
||
and assigns a value to the final key. Creates intermediate dictionaries
|
||
if they don't exist.
|
||
|
||
# Arguments
|
||
- `dict::Dict`
|
||
The root dictionary to traverse and modify.
|
||
- `accessArray::Array{Symbol}`
|
||
A vector of symbols representing the key path to traverse.
|
||
- `valueToAssign`
|
||
The value to assign at the final key in the path.
|
||
|
||
# Return
|
||
- `0` on success (value assigned)
|
||
- `1` if the path cannot be traversed (missing intermediate keys)
|
||
|
||
# Notes
|
||
- The function walks through each key in `accessArray` except the last one,
|
||
expecting intermediate keys to already exist in the dictionary.
|
||
- If any intermediate key is missing, the function returns `1` without
|
||
modifying the dictionary.
|
||
- The final key in `accessArray` receives the `valueToAssign`.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> d = Dict(
|
||
:a1=> Dict(:c=> 5),
|
||
:a2=> Dict(
|
||
:k=> 10,
|
||
:b=> Dict(
|
||
:s=> "target",
|
||
)
|
||
)
|
||
)
|
||
julia> assignDict!(d, [:a2, :b, :s], "wow")
|
||
0
|
||
julia> d[:a2][:b][:s]
|
||
"wow"
|
||
```
|
||
"""
|
||
function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign)
|
||
wd = nothing
|
||
for i in accessArray
|
||
println(i)
|
||
if i != accessArray[end]
|
||
if wd === nothing && haskey(dict, i)
|
||
wd = Ref(dict[i])
|
||
elseif wd.x !== nothing && haskey(wd.x, i)
|
||
wd = Ref(wd.x[i])
|
||
else
|
||
return 1 # error, no target key in a given dict.
|
||
end
|
||
else
|
||
wd.x[i] = valueToAssign
|
||
return 0
|
||
end
|
||
end
|
||
end
|
||
|
||
""" Converts hour (0-23) and minute (0-59) into a Julia `Time` object using
|
||
12-hour format with AM/PM indicator.
|
||
|
||
# Arguments
|
||
- `h::Integer`
|
||
Hour in 24-hour format (0 to 23).
|
||
- `m::Integer`
|
||
Minute (0 to 59).
|
||
|
||
# Return
|
||
- A `Time` object representing the time in 12-hour format with AM/PM.
|
||
|
||
# Notes
|
||
- Hours 0 and 12 are special cases: 0 becomes 12 AM, 12 becomes 12 PM.
|
||
- Hours 1-11 remain the same with "am" suffix.
|
||
- Hours 13-23 are converted to 1-11 with "pm" suffix.
|
||
- Minutes less than 10 are zero-padded.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> iTime(0, 30)
|
||
12:30 AM
|
||
|
||
julia> iTime(9, 15)
|
||
9:15 AM
|
||
|
||
julia> iTime(12, 0)
|
||
12:00 PM
|
||
|
||
julia> iTime(14, 5)
|
||
2:05 PM
|
||
```
|
||
"""
|
||
function iTime(h::Integer, m::Integer)
|
||
if h == 0
|
||
h = 12
|
||
ampm = "am"
|
||
elseif 1 <= h <= 11
|
||
ampm = "am"
|
||
elseif h == 12
|
||
ampm = "pm"
|
||
elseif 13 <= h <= 23
|
||
h = h - 12
|
||
ampm = "pm"
|
||
else
|
||
error("hour out of range")
|
||
end
|
||
|
||
m = m < 10 ? "0$m" : m
|
||
t = "$h:$m$ampm"
|
||
|
||
return Time(t, "HH:MMp")
|
||
end
|
||
|
||
""" replace a number according to the limit
|
||
if value is lower than lowerbound return lowerbound replacement value
|
||
if value is more than upperbound return upperbound replacement value
|
||
|
||
# Example
|
||
limitvalue(4, (-5 => 0), (5 => 5))
|
||
"""
|
||
function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair)
|
||
lwLimit, lwReplace = lowerbound
|
||
upLimit, upReplace = upperbound
|
||
|
||
if v < lwLimit
|
||
v = lwReplace
|
||
elseif v > upLimit
|
||
v = upReplace
|
||
else
|
||
end
|
||
return v
|
||
end
|
||
|
||
|
||
""" Assigns elements from matrix `b` to matrix `a` using the Cartesian indices
|
||
of `b`. Elements are copied in the order they appear when iterating over `b`,
|
||
and placed into `a` at the corresponding Cartesian positions of `b`.
|
||
|
||
# Arguments
|
||
- `a`
|
||
Target matrix where values from `b` will be assigned.
|
||
- `b`
|
||
Source matrix whose Cartesian indices determine where values are placed in `a`.
|
||
|
||
# Return
|
||
- `nothing`
|
||
|
||
# Notes
|
||
- The function iterates through `b` in column-major order (Julia's default),
|
||
retrieving each element's Cartesian index and assigning it to the same
|
||
position in `a`.
|
||
- Matrix `a` must have sufficient size to accommodate all Cartesian indices
|
||
from `b`; otherwise, an `BoundsError` may occur.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> a = zeros(4, 4);
|
||
julia> b = [1 2; 3 4];
|
||
julia> cartesianAssign!(a, b);
|
||
julia> a[1:2, 1:2]
|
||
2×2 Matrix{Float64}:
|
||
1.0 3.0
|
||
2.0 4.0
|
||
```
|
||
"""
|
||
function cartesianAssign!(a, b)
|
||
for (i, v) in enumerate(b)
|
||
a[CartesianIndices(b)[i].I...] = v
|
||
end
|
||
return nothing
|
||
end
|
||
|
||
|
||
function sumAlongDim3(a::Array)
|
||
totalDim = length(size(a))
|
||
|
||
if totalDim == 3
|
||
d1, d2, d3 = size(a)
|
||
r = zeros(1, 1, d3)
|
||
for i in 1:d3
|
||
view(r, 1, 1, i) .= sum(a[:, :, i])
|
||
end
|
||
elseif totalDim == 4
|
||
d1, d2, d3, d4 = size(a)
|
||
r = zeros(1, 1, d3, d4)
|
||
for j in 1:d4
|
||
for i in 1:d3
|
||
view(r, 1, 1, i, j) .= sum(a[:, :, i, j])
|
||
end
|
||
end
|
||
else
|
||
error("this condition is not define yet")
|
||
end
|
||
|
||
return r
|
||
end
|
||
|
||
|
||
""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all slice of 3D another matrix ,b, and
|
||
concatenate at the 4th dimension.
|
||
|
||
Example
|
||
julia> input = rand(32, 32, 128) # batch at 3rd dim
|
||
julia> weight = rand(32, 32, 1024)
|
||
julia> r = matMul3Dto3DmanyTo1batch(input, weight);
|
||
julia> size(r)
|
||
(32, 32, 1024, 128)
|
||
"""
|
||
function matMul3Dto3DmanyTo1batch(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing)
|
||
asize = [size(a)...]
|
||
bsize = [size(b)...]
|
||
if resultStorage === nothing
|
||
resultStorage = similar(a, eltype(b), bsize[1], bsize[2], bsize[3], asize[3])
|
||
end
|
||
|
||
c = [slice .* b for slice in eachslice(a, dims=3)]
|
||
resultStorage .= cat(c..., dims=4)
|
||
|
||
return resultStorage
|
||
end
|
||
|
||
|
||
""" GPU kernel
|
||
"""
|
||
function matMul3Dto3DmanyTo1batch_gpu!(a, b, resultStorage, linearToCartesian)
|
||
i = (blockIdx().x - 1) * blockDim().x + threadIdx().x
|
||
if i <= size(a, 3) # guard against unused threads to accessing memory out of bound
|
||
cartesianIndex = linearToCartesian(i, size(b)) # example for how to send "inner" function to gpu
|
||
# @cuprintln("gpu thread $i $cartesianIndex[2]")
|
||
|
||
@. @views resultStorage[:, :, :, i] = a[ :, :, i] * b
|
||
# view(resultStorage, :, :, :, i) .= view(a, :, :, i) .* b # alternative code
|
||
# @view(resultStorage[:, :, :, i]) .= @view(a[ :, :, i]) .* b # alternative code
|
||
end
|
||
return nothing
|
||
end
|
||
|
||
|
||
""" ELEMENT-wise multiply of each slice of 3D input matrix ,a, to all batch of another 4D matrix ,b, and
|
||
concatenate at the 4th dimension.
|
||
|
||
Example
|
||
julia>
|
||
julia> a = rand(2,2,3) # 3-batches
|
||
julia> b = rand(2,2,4,3) # 3-batches
|
||
julia> r = GeneralUtils.matMul_3Dto4D_batchwise(a, b);
|
||
julia> size(r)
|
||
(2, 2, 4, 3)
|
||
"""
|
||
function matMul_3Dto4D_batchwise(a::Array, b::Array; resultStorage::Union{Array, Nothing}=nothing)
|
||
if size(a, 3) != size(b, 4)
|
||
error("batch number of a and b must be equal")
|
||
end
|
||
if resultStorage === nothing
|
||
resultStorage = zeros(size(b, 1), size(b, 2), size(b, 3), size(a, 3))
|
||
end
|
||
for i in 1:size(a, 3)
|
||
view(resultStorage, :, :, :, i) .= a[:, :, i] .* b[:, :, :, i]
|
||
end
|
||
return resultStorage
|
||
end
|
||
|
||
""" GPU-compatible linear index to cartesian index conversion
|
||
"""
|
||
function linearToCartesian(i::Int, arraySize::NTuple{4,Int})
|
||
# Check that the linear coordinate is valid
|
||
# prod(arraySize) is the same as *(arraySize...). they multipy all elements in an array.
|
||
# but this code use prod() because splat breaks GPU performance
|
||
if i < 1 || i > prod(arraySize)
|
||
error("Invalid linear coordinate")
|
||
end
|
||
# Extract the dimensions of the matrix
|
||
n1, n2, n3, n4 = arraySize
|
||
# Compute the cartesian coordinate using rem and div functions
|
||
i1 = ((i-1) % (n1)) + 1 # +1 convert 0-based to 1-based index
|
||
i2 = ((i-1) ÷ (n1)) % n2 + 1
|
||
i3 = ((i-1) ÷ (n1*n2)) % n3 + 1
|
||
i4 = (i-1) ÷ (n1*n2*n3) + 1
|
||
# Return the cartesian coordinate as a tuple
|
||
return (i1, i2, i3, i4)
|
||
end
|
||
|
||
""" return a vector with true at max value and false for other value.
|
||
if vector is all-zeros, return all-false vector.
|
||
"""
|
||
function vectorMax(x)
|
||
if sum(isNotEqual.(x, 0)) == 0 # guard against all-zeros array
|
||
# instead of returning all-zeros original vector,
|
||
# return all-false vector to prevent type instability
|
||
return isNotEqual.(x, 0)
|
||
else
|
||
return isequal.(x, maximum(x))
|
||
end
|
||
end
|
||
|
||
function multiply_last(matrix, x, n)
|
||
# X is the scalar to multiply
|
||
# matrix is the column-major 2D matrix
|
||
# n is the number of elements to be multiplied, starting from the last one
|
||
# returns a new matrix with the same shape as the original one
|
||
|
||
# get the number of rows and columns of the matrix
|
||
rows, cols = size(matrix)
|
||
|
||
# create a copy of the matrix to avoid mutating the original one
|
||
result = copy(matrix)
|
||
|
||
# loop over the last n elements in column-major order
|
||
for i in (rows * cols - n + 1):(rows * cols)
|
||
# get the row and column indices of the current element
|
||
row = (i - 1) % rows + 1
|
||
col = (i - 1) ÷ rows + 1
|
||
|
||
# multiply the element by X and store it in the result matrix
|
||
result[row, col] *= x
|
||
end
|
||
|
||
# return the result matrix
|
||
return result
|
||
end
|
||
|
||
function multiplyRandomElements(A, x, n, rng=MersenneTwister(1234))
|
||
# rng is a random number generator object, see https://docs.julialang.org/en/v1/stdlib/Random/
|
||
# x is a scalar value to multiply by
|
||
# A is a column-major 2D matrix or a vector
|
||
# n is the number of elements to be multiplied
|
||
# returns a new array with n randomly chosen distinct elements multiplied by x
|
||
B = copy(A) # make a copy of A to avoid mutating it
|
||
d = ndims(A) # get the number of dimensions of A
|
||
if d == 1 # if A is a vector
|
||
m = length(A) # get the length of A
|
||
indices = collect(1:m) # create an array of indices from 1 to m
|
||
shuffle!(rng, indices) # shuffle the indices in-place using the RNG
|
||
for i in 1:n # loop n times
|
||
j = indices[i] # get the i-th shuffled index
|
||
B[j] *= x # multiply the element at j by x
|
||
end
|
||
elseif d == 2 # if A is a matrix
|
||
m = size(A, 1) # number of rows in A
|
||
p = size(A, 2) # number of columns in A
|
||
indices = collect(1:m*p) # create an array of linear indices from 1 to m*p
|
||
shuffle!(rng, indices) # shuffle the indices in-place using the RNG
|
||
for i in 1:n # loop n times
|
||
j = indices[i] # get the i-th shuffled index
|
||
B[j] *= x # multiply the element at j by x
|
||
end
|
||
else # if A is neither a vector nor a matrix
|
||
error("A must be a vector or a matrix")
|
||
end
|
||
return B # return the new array
|
||
end
|
||
|
||
""" Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and
|
||
replace matrix A's elements of the same position with value, a.
|
||
|
||
Example
|
||
julia> mask = rand([-1,0,1],4,4,1)
|
||
julia> A = rand(4,4,1)
|
||
julia> C = replaceElements(mask, A, -1, 5.0, 3)
|
||
"""
|
||
function replaceElements(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number,
|
||
n::Int=0; rng::AbstractRNG=MersenneTwister(1234))
|
||
""" Prompt
|
||
Write a julia function to operate on column-major 3D matrix. The function randomly
|
||
choose elements in matrix mask that has value markValue and replace elements in matrix A at
|
||
the same position with value a. The choosing randomness is controlled by rng function.
|
||
I also want to specify how many elements to be replaced.
|
||
"""
|
||
|
||
total_x_tobeReplced = sum(isequal.(mask, markValue))
|
||
if n == 0 || n > total_x_tobeReplced
|
||
n = total_x_tobeReplced
|
||
end
|
||
|
||
# check if mask and A have the same size
|
||
if size(mask) != size(A)
|
||
error("mask and A must have the same size")
|
||
end
|
||
C = copy(A)
|
||
# get the indices of elements in mask that equal markValue
|
||
indices = findall(x -> x == markValue, mask)
|
||
# shuffle the indices using the rng function
|
||
shuffle!(rng, indices)
|
||
# select the first n indices
|
||
selected = indices[1:n]
|
||
# replace the elements in A at the selected positions with a
|
||
for i in selected
|
||
C[i] = a
|
||
end
|
||
return C
|
||
end
|
||
|
||
""" Randomly (rng controlled) choose position of elements that has value, markValue, from matrix mask and
|
||
replace matrix A's elements of the same position with value, a. if n == 0, all marked value is replaced
|
||
|
||
Example
|
||
julia> mask = rand([-1,0,1],4,4,1)
|
||
julia> A = rand(4,4,1)
|
||
julia> C = replaceElements(mask, A, -1, 5.0, 3)
|
||
"""
|
||
function replaceElements!(mask::AbstractArray{<:Any}, markValue::Number, A::AbstractArray{<:Any}, a::Number,
|
||
n::Int=0; rng::AbstractRNG=MersenneTwister(1234))
|
||
total_x_tobeReplced = sum(isequal.(mask, markValue))
|
||
remaining = 0
|
||
if n == 0 || n > total_x_tobeReplced
|
||
remaining = n - total_x_tobeReplced
|
||
n = total_x_tobeReplced
|
||
end
|
||
|
||
# check if mask and A have the same size
|
||
if size(mask) != size(A)
|
||
error("mask and A must have the same size, mask $(size(mask)) A $(size(A))")
|
||
end
|
||
# get the indices of elements in mask that equal markValue
|
||
indices = findall(x -> x == markValue, mask)
|
||
# shuffle the indices using the rng function
|
||
shuffle!(rng, indices)
|
||
# select the first n indices
|
||
selected = indices[1:n]
|
||
# replace the elements in A at the selected positions with a
|
||
for i in selected
|
||
A[i] = a
|
||
end
|
||
|
||
return remaining
|
||
end
|
||
|
||
""" Replace n elements that has value x with user specified value a.
|
||
"""
|
||
function replaceElements(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234))
|
||
total_x_tobeReplced = sum(isequal.(A, x))
|
||
if n == 0 || n > total_x_tobeReplced
|
||
n = total_x_tobeReplced
|
||
end
|
||
|
||
B = copy(A)
|
||
# A is a column-major 3D matrix
|
||
# x is the value to be replaced
|
||
# a is the new value
|
||
# rng is a random number generator function
|
||
# n is the number of elements to be replaced
|
||
|
||
# find the indices of elements in A that equal x
|
||
indices = findall(==(x), B)
|
||
|
||
# shuffle the indices using the rng function
|
||
shuffle!(rng, indices)
|
||
|
||
# select the first n indices
|
||
selected = indices[1:n]
|
||
|
||
# replace the elements at the selected indices with a
|
||
for i in selected
|
||
B[i] = a
|
||
end
|
||
|
||
# return the modified matrix A
|
||
return B
|
||
end
|
||
|
||
|
||
function replaceElements!(A::AbstractArray{<:Any}, x::Number, a::Number, n::Int=0, rng=MersenneTwister(1234))
|
||
total_x_tobeReplced = sum(isequal.(A, x))
|
||
if n == 0 || n > total_x_tobeReplced
|
||
n = total_x_tobeReplced
|
||
end
|
||
|
||
# A is a column-major 3D matrix
|
||
# x is the value to be replaced
|
||
# a is the new value
|
||
# rng is a random number generator function
|
||
# n is the number of elements to be replaced
|
||
|
||
# find the indices of elements in A that equal x
|
||
indices = findall(==(x), A)
|
||
|
||
# shuffle the indices using the rng function
|
||
shuffle!(rng, indices)
|
||
|
||
# select the first n indices
|
||
selected = indices[1:n]
|
||
|
||
# replace the elements at the selected indices with a
|
||
for i in selected
|
||
A[i] = a
|
||
end
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
""" Get characters between specified characters.
|
||
|
||
# Arguments
|
||
- `text::T`
|
||
a text being searched
|
||
- `startChar::Char`
|
||
start character
|
||
- `endChar::Char`
|
||
end character
|
||
# Keyword Arguments
|
||
- `endCharLocation::String`
|
||
end character position after startChar. Can be "next" or "end". "next" means the closed
|
||
endChar just after startChar. "end" means the furthest endChar.
|
||
- `includeChar::Bool`
|
||
whether to include the startChar and endChar. Default is true
|
||
# Return
|
||
the characters between specified characters.
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> using Revise
|
||
julia> using GeneralUtils
|
||
julia> text = "{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}\n\n"
|
||
julia> GeneralUtils.getStringBetweenCharacters(text, '{', '}', endCharLocation="end")
|
||
"{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}"
|
||
```
|
||
"""
|
||
function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char;
|
||
endCharLocation::String="next", includeChar::Bool=true)::String where {T<:AbstractString}
|
||
|
||
# get the position of the startChar
|
||
startCharPosition = findfirst(startChar, text)
|
||
endCharPosition = nothing
|
||
|
||
if endCharLocation == "end"
|
||
# get the first position of the endChar coming from the end of text
|
||
endCharPosition = findlast(endChar, text)
|
||
elseif endCharLocation == "next"
|
||
# get the first position of the endChar after startCharPosition
|
||
endCharPosition = findnext(endChar, text, startCharPosition + 1)
|
||
else
|
||
error("endCharPositio must be \"end\" or \"next\"")
|
||
end
|
||
|
||
@show startCharPosition, endCharPosition
|
||
|
||
# get characters between startChar and endChar from text
|
||
extractedText = text[startCharPosition:endCharPosition]
|
||
# convert substring to string
|
||
extractedText = string(extractedText)
|
||
|
||
extractedText = includeChar == true ? extractedText : extractedText[2:end-1]
|
||
|
||
return extractedText
|
||
end
|
||
|
||
|
||
""" Recursively creates nested dictionary paths if they do not exist and assigns
|
||
a value to the final key. Similar to `mkpath()` but for dictionaries.
|
||
|
||
# Arguments
|
||
- `dict::Union{Dict{Symbol, Any}, Dict{String, Any}}`
|
||
The target dictionary to traverse and modify. Must use consistent key types
|
||
(either all `String` or all `Symbol`).
|
||
- `addkeys::Union{Vector{String}, Vector{Symbol}}`
|
||
A vector of keys representing the path to traverse. Intermediate dictionaries
|
||
are created if they don't exist.
|
||
- `value`
|
||
The value to assign at the final key in the path.
|
||
|
||
# Return
|
||
- The assigned `value`.
|
||
|
||
# Notes
|
||
- The function ensures key type consistency: the type of keys being added must
|
||
match the type of existing keys in the dictionary.
|
||
- Intermediate dictionaries are automatically created with the appropriate key
|
||
type when they don't exist.
|
||
- The function walks through each key in `addkeys` except the last one,
|
||
creating intermediate dictionaries as needed, and assigns `value` to the final
|
||
key.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> d = Dict("a" => Dict("b" => 10))
|
||
julia> mkDictPath!(d, ["a", "v", "x", "y", "z"], 42)
|
||
42
|
||
julia> d["a"]["v"]["x"]["y"]["z"]
|
||
42
|
||
```
|
||
"""
|
||
function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}},
|
||
addkeys::Union{Vector{String}, Vector{Symbol}}, value)
|
||
# new key and existing key must be the same type
|
||
if !isempty(dict)
|
||
existingKeys = [key for key in keys(dict)]
|
||
if typeof(existingKeys[1]) != typeof(addkeys[1])
|
||
error("Type of keys being added is $(typeof(addkeys[1])) but type of existing keys is $(typeof(existingKeys[1]))")
|
||
end
|
||
end
|
||
|
||
for key in addkeys[1:end-1]
|
||
if !haskey(dict, key)
|
||
key_type = eltype(keys(dict))
|
||
dict[key] = Dict{key_type, Any}()
|
||
end
|
||
dict = dict[key]
|
||
end
|
||
|
||
return dict[addkeys[end]] = value
|
||
end
|
||
|
||
|
||
""" Retrieves a value from a nested dictionary by traversing a vector of keys.
|
||
Creates intermediate dictionaries if they don't exist.
|
||
|
||
# Arguments
|
||
- `dict::Dict`
|
||
The root dictionary to traverse.
|
||
- `keys::Vector`
|
||
A vector of keys representing the path to traverse. Each key in the vector
|
||
is used to access the next level of nesting.
|
||
|
||
# Return
|
||
- The value at the final key in the path.
|
||
|
||
# Notes
|
||
- Errors with `ArgumentError` if any intermediate key is missing from the
|
||
dictionary path.
|
||
- The function walks through each key in `keys` except the last one,
|
||
expecting intermediate keys to exist in the dictionary.
|
||
- The final key in `keys` is used to retrieve the value.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> using GeneralUtils
|
||
julia> d = Dict(:a => Dict(:b => 10))
|
||
julia> getDictPath(d, [:a, :b])
|
||
10
|
||
```
|
||
"""
|
||
function getDictPath(dict::Dict, keys::Vector)
|
||
current_dict = dict
|
||
for key in keys[1:end-1]
|
||
if haskey(current_dict, key)
|
||
current_dict = current_dict[key]
|
||
else
|
||
throw(ArgumentError("Key $key not found in dictionary"))
|
||
end
|
||
end
|
||
|
||
last_key = keys[end]
|
||
if haskey(current_dict, last_key)
|
||
return current_dict[last_key]
|
||
else
|
||
throw(ArgumentError("Key $last_key not found in dictionary"))
|
||
end
|
||
end
|
||
|
||
|
||
|
||
"""
|
||
detectKeywordVariation(keywords::AbstractVector{String}, text::String) -> Dict{String, Union{Array, Nothing}}
|
||
|
||
Detects and collects all case-variant occurrences of multiple keywords in the text.
|
||
This function processes each keyword individually and returns an array of matched variations for each keyword.
|
||
|
||
# Arguments
|
||
- `keywords::AbstractVector{String}` Vector of keywords to search for
|
||
- `text::String` The text to search in
|
||
|
||
# Returns
|
||
- `Dict{String, Array}` Returns a dictionary mapping each keyword to an array of matched variations found in the text
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> detectKeywordVariation(["test", "example", "cat"], "This is a Test EXAMPLE")
|
||
Dict{String, Array}("test" => ["Test"], "example" => ["EXAMPLE"], "cat" => nothing)
|
||
"""
|
||
function detectKeywordVariation(keywords::T, text::String)::Dict{String, Union{Array, Nothing}} where {T<:AbstractVector}
|
||
kw = Dict{String, Union{Array, Nothing}}()
|
||
|
||
# use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
|
||
for keyword in keywords
|
||
ws = detectKeywordVariation.(keyword, text)
|
||
total = sum(issomething.(ws))
|
||
if total != 0
|
||
kw[keyword] = ws
|
||
else
|
||
kw[keyword] = nothing
|
||
end
|
||
end
|
||
return kw
|
||
end
|
||
|
||
|
||
"""
|
||
detectKeywordVariation(keyword::String, text::String) -> Union{Nothing, Array{String}}
|
||
|
||
Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase).
|
||
|
||
# Arguments:
|
||
- `keyword::String` The keyword to search for
|
||
- `text::String` The text to search in
|
||
|
||
# Returns:
|
||
- `Union{Nothing, Array{String}}` Returns an array of matched keyword variations if found, otherwise returns nothing
|
||
|
||
# Examples:
|
||
```jldoctest
|
||
julia> detectKeywordVariation("test", "This is a Test case")
|
||
["Test"]
|
||
|
||
julia> detectKeywordVariation("error", "NO ERRORS FOUND")
|
||
["ERRORS"]
|
||
|
||
julia> detectKeywordVariation("missing", "complete data")
|
||
nothing
|
||
```
|
||
"""
|
||
function detectKeywordVariation(keyword::String, text::String)::Union{Nothing, Array{String}}
|
||
# Define the keyword variations to search for
|
||
wordVariations = [uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)]
|
||
# wordVariations may duplicate keyword
|
||
keyword_variations = [keyword]
|
||
for i in wordVariations
|
||
i != keyword ? push!(keyword_variations, i) : nothing
|
||
end
|
||
|
||
_splittext = string.(strip.(split(text, " ")))
|
||
splittext = String[]
|
||
# remove . after a word
|
||
for i in _splittext
|
||
if length(i) != 0 && i[end] ∈ ['.']
|
||
word = string(i[1:end-1])
|
||
push!(splittext, word)
|
||
else
|
||
push!(splittext, i)
|
||
end
|
||
end
|
||
|
||
result = String[]
|
||
for variation in keyword_variations
|
||
# if length of both word is equals then it is a whole word otherwise it is part of part of other word
|
||
r = findIndex(splittext, variation)
|
||
|
||
if isempty(r[2])
|
||
# skip
|
||
else
|
||
# if variation > 1 add them all so this function detect duplicate keyword
|
||
variations = [variation for i in eachindex(r[2])]
|
||
result = vcat(result, variations)
|
||
end
|
||
end
|
||
return result
|
||
end
|
||
|
||
|
||
""" Convert text into a dictionary with a given keywords. This function use keywords to slice
|
||
a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text.
|
||
The left most string which has no keyword will be discarded. WARNING, ordering is important
|
||
|
||
# Arguments
|
||
- `text::String`
|
||
A text to be converted.
|
||
- `keywords::Vector{String}`
|
||
A list of keywords to be used to slice the text.
|
||
These keywords also be the resulting dict keys.
|
||
# Keyword Arguments
|
||
- `rightmarker::String`
|
||
A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":",
|
||
the function will search for "plan:" otherwise the function will search for "plan".
|
||
The marker will not be in the resulting dict keys.
|
||
- `symbolkey::Bool`
|
||
If true, resulting dict's key will be Symbols, otherwise string.
|
||
- `lowercasekey::Bool`
|
||
set resulting dict's key to be lowercase
|
||
|
||
# Return
|
||
- `d::OrderedDict`
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep"
|
||
julia> sample_keywords = ["thought", "plan", "action"]
|
||
julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true)
|
||
julia> println(resultdict)
|
||
OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do",
|
||
:plan => "wake up and going out",
|
||
:action => "1. wake up 2. eat 3. sleep")
|
||
```
|
||
"""
|
||
function textToDict(text::String, detectKeywords::Vector{String};
|
||
dictKey::Union{Vector{String}, Nothing}=nothing,
|
||
symbolkey::Bool=false, lowercasekey::Bool=false
|
||
)::OrderedDict
|
||
|
||
# make sure this function detect variation of a work e.g. agent, Agent, AGENT
|
||
kw = []
|
||
# use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
|
||
for keyword in detectKeywords
|
||
detected = detectKeywordVariation(keyword, text)
|
||
if detected !== nothing
|
||
push!(kw, detected)
|
||
else
|
||
error("Keyword $keyword not found in text: $text")
|
||
end
|
||
end
|
||
if typeof(kw[1]) <: AbstractArray
|
||
kw = reduce(vcat, kw)
|
||
end
|
||
|
||
od1, od2 =
|
||
if symbolkey
|
||
OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}()
|
||
else
|
||
OrderedDict{String, Any}(), OrderedDict{String, Any}()
|
||
end
|
||
|
||
remainingtext = text
|
||
dictKey_ = reverse(dictKey)
|
||
|
||
# process text from back to front
|
||
rkw = reverse(kw)
|
||
for (i,keyword) in enumerate(rkw)
|
||
# Find the position of the keyword in the text
|
||
keywordidx = findlast(keyword, remainingtext)
|
||
dKey = dictKey_[i]
|
||
|
||
if keywordidx !== nothing
|
||
substr = remainingtext[keywordidx[end]+1:end]
|
||
str = string(strip(substr)) # Removes both leading and trailing whitespace.
|
||
_key = lowercasekey == true ? lowercase(dKey) : dKey
|
||
key = symbolkey == true ? Symbol(_key) : _key
|
||
od1[key] = str
|
||
remainingtext = remainingtext[1:keywordidx[1]-1]
|
||
else
|
||
error("""keyword "$keyword" not found in the provided text: $text </end of error note>""")
|
||
end
|
||
end
|
||
|
||
# correct the order
|
||
ks = reverse([i for i in keys(od1)])
|
||
for k in ks
|
||
k = symbolkey == true ? Symbol(k) : k
|
||
od2[k] = od1[k]
|
||
end
|
||
|
||
return od2
|
||
end
|
||
|
||
""" Recursively convert dictionary into an HTML string representation.
|
||
The function walks a nested `AbstractDict` structure and produces a well-formed HTML string
|
||
where each dictionary key becomes an HTML tag. Nested dictionaries become
|
||
nested tags, and scalar values (numbers, strings, etc.) become the text
|
||
content of leaf tags.
|
||
|
||
# Arguments
|
||
- `d::AbstractDict`
|
||
The dictionary to convert. Keys must be strings or symbols that form valid
|
||
HTML tag names.
|
||
|
||
# Keyword Arguments
|
||
- `indent_level::Integer=1`
|
||
Initial indentation level for the output. Each recursive level increases
|
||
indentation by one.
|
||
- `indent_str::String=" "`
|
||
String used for each indentation level.
|
||
|
||
# Return
|
||
- A single HTML string representing the dictionary structure, with proper
|
||
opening and closing tags and appropriate indentation.
|
||
|
||
# Notes
|
||
- Keys are sorted alphabetically for deterministic output.
|
||
- Works recursively: dictionary values produce nested tags; non-dictionary
|
||
values are placed as text between opening/closing tags.
|
||
|
||
# Examples
|
||
```jldoctest
|
||
julia> d = Dict(
|
||
"html" => Dict(
|
||
"head" => Dict("title" => "Test"),
|
||
"body" => Dict(
|
||
"h1" => "Hello",
|
||
"p" => "World"
|
||
)
|
||
)
|
||
);
|
||
|
||
julia> println(dict_to_string_html(d))
|
||
<html>
|
||
<head>
|
||
<title>Test</title>
|
||
</head>
|
||
<body>
|
||
<h1>Hello</h1>
|
||
<p>World</p>
|
||
</body>
|
||
</html>
|
||
```
|
||
"""
|
||
function dict_to_string_html(d::AbstractDict; indent_level=1, indent_str=" ")
|
||
lines = String[]
|
||
padding = indent_str ^ indent_level
|
||
|
||
for k in keys(d)
|
||
v = d[k]
|
||
|
||
if v isa AbstractDict
|
||
# Open tag, recurse for children, then close tag
|
||
push!(lines, "$padding<$k>")
|
||
ind_level = indent_level + 1
|
||
push!(lines, dict_to_string_html(v; indent_level=ind_level, indent_str=indent_str))
|
||
push!(lines, "$padding</$k>")
|
||
else
|
||
# Leaf node: put key and value on a single line
|
||
push!(lines, "$padding<$k>$v</$k>")
|
||
end
|
||
end
|
||
return join(lines, "\n")
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
end # module |