Compare commits
57 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 517a08365d | |||
| d032cec0b2 | |||
| 64575d81b5 | |||
| fb54fddf10 | |||
| 407447831a | |||
| c475eb169c | |||
| d1a279cca2 | |||
| bf3b65ee7b | |||
| 360d64c474 | |||
| 41a354fa73 | |||
| 801596fa7f | |||
| 2fbe9d6e1a | |||
| 2c2690e5dd | |||
| 95db5f877d | |||
| 7e2ddd846e | |||
| ab113acde5 | |||
| 7391f0f2ce | |||
| 01f4e52c64 | |||
| 35acfe5b70 | |||
| e1ebbf370e | |||
| 1fd3fa1bee | |||
| 9721a393bc | |||
| 1db8e4e383 | |||
| c51dfc549c | |||
| f0ad6a3e48 | |||
| 07d5d0f885 | |||
| c829bf65f6 | |||
| b09efc9068 | |||
| 6fb2d5f82b | |||
| 55cc0f78c9 | |||
| 73ec3bbb04 | |||
| 1ad46c6e18 | |||
| 95954249ce | |||
| a15630619a | |||
| f28405f3f1 | |||
| d658d9a25b | |||
| edad442242 | |||
| c56fc7366c | |||
| c4eeb99aba | |||
| bd5022c8bc | |||
| db84b1c398 | |||
| f8f8410259 | |||
| 956adf0b93 | |||
| 0f2a33bcdd | |||
| b8f84846bb | |||
| adf6264061 | |||
| 08f19f17a2 | |||
| 8a4e882dc1 | |||
| fb7942a965 | |||
| e54454b099 | |||
| dac98ab38c | |||
| 1e8149aa6f | |||
| 8f12c29a78 | |||
| e3d09e6ebd | |||
| f33f4f0790 | |||
| fcf2044dd9 | |||
| 05d8cb9c02 |
+543
-112
File diff suppressed because it is too large
Load Diff
+14
-2
@@ -1,23 +1,35 @@
|
||||
name = "GeneralUtils"
|
||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||
version = "0.3.2"
|
||||
version = "0.6.1"
|
||||
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
||||
|
||||
[deps]
|
||||
AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
|
||||
AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
|
||||
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
|
||||
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
||||
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
||||
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
||||
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
|
||||
Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
|
||||
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
||||
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
|
||||
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
||||
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
||||
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
||||
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
||||
StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
|
||||
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||
|
||||
[compat]
|
||||
JSON = "1.3.0"
|
||||
AWS = "1.99.0"
|
||||
AWSS3 = "0.11.6"
|
||||
Graphs = "1.14.0"
|
||||
HTTP = "2.5.0 - 2.9.9"
|
||||
JSON = "1.3.0 - 1.9.9"
|
||||
LibPQ = "1.18.0"
|
||||
NATS = "0.1.0"
|
||||
Revise = "3.13.2"
|
||||
StringDistances = "1.0.0"
|
||||
|
||||
@@ -21,6 +21,8 @@ using .llmUtil
|
||||
include("interface.jl")
|
||||
using .interface
|
||||
|
||||
include("garageS3.jl")
|
||||
using .garageS3
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
|
||||
+96
-2
@@ -1,13 +1,107 @@
|
||||
module dbUtil
|
||||
|
||||
export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
|
||||
export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
|
||||
execute_postgres_sql
|
||||
|
||||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
|
||||
SHA
|
||||
SHA, NATS, LibPQ
|
||||
using ..util
|
||||
#[PENDING] update code to use JSON
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" Execute SQL against a PostgreSQL database using LibPQ connection string.
|
||||
|
||||
# Arguments
|
||||
- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
|
||||
- `sql::AbstractString`: SQL query to execute
|
||||
|
||||
# Returns
|
||||
- `LibPQ.Result` on success, `nothing` on failure
|
||||
|
||||
# Example
|
||||
```julia
|
||||
pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
|
||||
sql = "SELECT * FROM wine;"
|
||||
result = execute_postgres_sql(pg_conn_str, sql)
|
||||
```
|
||||
"""
|
||||
function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
|
||||
db_connection = LibPQ.Connection(pg_conn_str)
|
||||
result = nothing
|
||||
try
|
||||
result = LibPQ.execute(db_connection, sql)
|
||||
catch e
|
||||
@error e
|
||||
LibPQ.close(db_connection)
|
||||
end
|
||||
|
||||
LibPQ.close(db_connection)
|
||||
return result
|
||||
end
|
||||
|
||||
|
||||
""" find_text_vector_similarity
|
||||
|
||||
Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
|
||||
|
||||
This function computes an embedding for the input text using the provided embedding function,
|
||||
then queries the database to find records with the most similar vector representations using
|
||||
PostgreSQL's cosine similarity operator (`<->`).
|
||||
|
||||
# Arguments
|
||||
- `text::AbstractString`: The input text to find similar records for
|
||||
- `tablename::AbstractString`: Name of the database table containing the embedding column
|
||||
- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
|
||||
- `executesql::Function`: Function that executes SQL queries and returns results
|
||||
- `get_embedding::Function`: Function that generates embeddings for text inputs
|
||||
|
||||
# Keyword Arguments
|
||||
- `limit::Integer=1`: Maximum number of similar records to return
|
||||
|
||||
# Returns
|
||||
- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
|
||||
where smaller values indicate higher similarity
|
||||
|
||||
# Example
|
||||
```julia
|
||||
# Assume you have embedding and SQL execution functions
|
||||
text = "a rich structured red wine"
|
||||
tablename = "wine"
|
||||
embeddingColumnName = "description_embedding"
|
||||
|
||||
df = find_text_vector_similarity(
|
||||
text, tablename, embeddingColumnName,
|
||||
executesql, get_embedding;
|
||||
limit = 5
|
||||
)
|
||||
|
||||
# Result contains columns from the table plus a 'distance' column
|
||||
# where distance = 1 - cosine_similarity (smaller = more similar)
|
||||
```
|
||||
"""
|
||||
function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
|
||||
executesql::Function, get_embedding;
|
||||
limit::Integer=1
|
||||
)::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
|
||||
# get embedding from LLM service
|
||||
_embedding = get_embedding([text])
|
||||
_embedding = _embedding["data"][1]["embedding"]
|
||||
_embedding = "$_embedding"
|
||||
|
||||
embedding = _embedding[4:end] # remove 'Any' from Any[...]
|
||||
|
||||
# check whether there is close enough vector already store in executesql. if no, add, else skip
|
||||
sql = """
|
||||
SELECT *, $embeddingColumnName <-> '$embedding' as distance
|
||||
FROM $tablename
|
||||
ORDER BY distance LIMIT $limit;
|
||||
"""
|
||||
response = executesql(sql)
|
||||
df = DataFrame(response)
|
||||
|
||||
return df
|
||||
end
|
||||
|
||||
|
||||
"""
|
||||
dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
|
||||
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
module garageS3
|
||||
|
||||
export
|
||||
GarageStorage,
|
||||
put_file,
|
||||
get_file,
|
||||
list_files,
|
||||
delete_file
|
||||
|
||||
using AWS, AWSS3
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
||||
# this file use local AWS config
|
||||
|
||||
""" Example
|
||||
const storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
||||
"sommpanion-s3"
|
||||
)
|
||||
|
||||
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
||||
put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
|
||||
keys = list_files(storage)
|
||||
data = String(get_file(storage, "users-1004.json"))
|
||||
|
||||
println("Read back: ", data)
|
||||
println("Bucket keys: ", keys)
|
||||
|
||||
# test with curl
|
||||
curl -v \
|
||||
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
||||
http://192.168.88.106:3902/users-1002.json
|
||||
|
||||
|
||||
# test with a browser
|
||||
https://s3-web.mydomain.com/my_bucket_name/users-1002.json
|
||||
|
||||
"""
|
||||
|
||||
# ===================================================================
|
||||
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
||||
# ===================================================================
|
||||
|
||||
struct SimpleCredentials
|
||||
access_key_id::String
|
||||
secret_key::String
|
||||
token::String
|
||||
end
|
||||
|
||||
struct GarageConfig <: AWS.AbstractAWSConfig
|
||||
endpoint::String
|
||||
region::String
|
||||
credentials::SimpleCredentials
|
||||
end
|
||||
|
||||
# Required extensions for AWS.jl pipeline
|
||||
AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
|
||||
AWS.credentials(c::SimpleCredentials) = c
|
||||
AWS.check_credentials(c::SimpleCredentials) = c
|
||||
AWS.region(aws::GarageConfig) = aws.region
|
||||
AWS.credentials(aws::GarageConfig) = aws.credentials
|
||||
|
||||
""" Generate the base URL for a Garage S3 service request.
|
||||
|
||||
# Arguments
|
||||
- `aws::GarageConfig`
|
||||
The Garage configuration containing the endpoint.
|
||||
- `service::String`
|
||||
The AWS service name (e.g., `"s3"`).
|
||||
- `region::String`
|
||||
The AWS region string.
|
||||
|
||||
# Return
|
||||
- `String`: The stripped endpoint URL (trailing slashes removed).
|
||||
"""
|
||||
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
||||
return strip(aws.endpoint, '/')
|
||||
end
|
||||
|
||||
""" Generate the full service URL including the resource path for a Garage S3 request.
|
||||
|
||||
# Arguments
|
||||
- `aws::GarageConfig`
|
||||
The Garage configuration containing the endpoint.
|
||||
- `service::String`
|
||||
The AWS service name (e.g., `"s3"`).
|
||||
- `resource::String`
|
||||
The S3 resource path (e.g., `"/bucket/key"`).
|
||||
|
||||
# Return
|
||||
- `String`: The full URL combining the endpoint and resource path.
|
||||
"""
|
||||
function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
|
||||
endpoint = strip(aws.endpoint, '/')
|
||||
resource_path = startswith(resource, '/') ? resource : "/" * resource
|
||||
return string(endpoint, resource_path)
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# 2. Thread-Safe Storage Client Wrapper
|
||||
# ===================================================================
|
||||
|
||||
struct GarageStorage
|
||||
config::GarageConfig
|
||||
bucket::String
|
||||
end
|
||||
""" Create a thread-safe Garage S3 storage client.
|
||||
|
||||
# Arguments
|
||||
- `endpoint::String`
|
||||
The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
|
||||
- `key::String`
|
||||
The Garage access key ID (created via garage-ui).
|
||||
- `secret::String`
|
||||
The secret key corresponding to the access key.
|
||||
- `bucket::String`
|
||||
The name of the S3 bucket to use.
|
||||
|
||||
# Keyword Arguments
|
||||
- `region::String = "garage"`
|
||||
The region identifier for the Garage instance.
|
||||
|
||||
# Return
|
||||
- `GarageStorage`: A thread-safe storage client instance.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "sommpanion-s3")
|
||||
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
||||
```
|
||||
"""
|
||||
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
||||
creds = SimpleCredentials(key, secret, "")
|
||||
config = GarageConfig(endpoint, region, creds)
|
||||
return GarageStorage(config, bucket)
|
||||
end
|
||||
|
||||
""" Upload an object to the Garage S3 bucket.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`
|
||||
The storage client instance.
|
||||
- `key::String`
|
||||
The object key (name) in the bucket. Must not contain slashes (`/`).
|
||||
- `data::Union{String, Vector{UInt8}}`
|
||||
The data to upload.
|
||||
|
||||
# Return
|
||||
- `Nothing` if the key contains slashes (upload aborted with warning).
|
||||
- Prints a success message if the upload completes.
|
||||
|
||||
# Notes
|
||||
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
||||
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> put_file(storage, "test.json", "{\"status\": \"active\"}")
|
||||
Successfully uploaded: test.json
|
||||
```
|
||||
"""
|
||||
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})
|
||||
# Check if the key contains a slash (virtual folder character)
|
||||
if occursin('/', key)
|
||||
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
||||
return nothing
|
||||
end
|
||||
|
||||
# Proceed if the key is flat
|
||||
s3_put(storage.config, storage.bucket, key, data)
|
||||
println("Successfully uploaded: ", key)
|
||||
end
|
||||
|
||||
""" Download an object from the Garage S3 bucket.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`
|
||||
The storage client instance.
|
||||
- `key::String`
|
||||
The object key (name) in the bucket.
|
||||
|
||||
# Return
|
||||
- `Vector{UInt8}`: The raw bytes of the downloaded object.
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> data = String(get_file(storage, "test.json"))
|
||||
"\"{\\\"status\\\": \\\"active\\\"}\""
|
||||
```
|
||||
"""
|
||||
function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
|
||||
return s3_get(storage.config, storage.bucket, key)
|
||||
end
|
||||
|
||||
""" List all object keys in the Garage S3 bucket.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`
|
||||
The storage client instance.
|
||||
|
||||
# Return
|
||||
- `Vector{String}`: A list of object keys (file names) in the bucket.
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> keys = list_files(storage)
|
||||
3-element Vector{String}:
|
||||
"test1.json"
|
||||
"test2.json"
|
||||
"users-1002.json"
|
||||
```
|
||||
"""
|
||||
function list_files(storage::GarageStorage)
|
||||
# Approach 2: s3_list_objects returns a Vector of Dicts with object details
|
||||
objects = s3_list_objects(storage.config, storage.bucket)
|
||||
return [obj["Key"] for obj in objects]
|
||||
end
|
||||
|
||||
""" Delete an object from the Garage S3 bucket.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`
|
||||
The storage client instance.
|
||||
- `key::String`
|
||||
The object key (name) to delete.
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> delete_file(storage, "test.json")
|
||||
```
|
||||
"""
|
||||
function delete_file(storage::GarageStorage, key::String)
|
||||
s3_delete(storage.config, storage.bucket, key)
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module GarageS3
|
||||
+463
-206
@@ -5,7 +5,7 @@ export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue,
|
||||
replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul3Dto3DmanyTo1batch,
|
||||
matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax,
|
||||
multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween,
|
||||
isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!,
|
||||
isLess, allTrue, getStringBetweenCharacters, mkDictPath!, dict_to_string_html,
|
||||
getDictPath, detectKeywordVariation, textToDict, dictify, ordereddictify
|
||||
|
||||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV
|
||||
@@ -150,14 +150,31 @@ end
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" read_textfile_by_index(folder_path::String, read_file_number::Integer=1)
|
||||
""" Reads the x-th text file from a folder, where files are listed by the OS
|
||||
without explicit sorting. Returns the file number, filename, and content.
|
||||
|
||||
with multiple text file in a folder,
|
||||
this function read x_th text file in a folder (filename is sorted by OS)
|
||||
# Arguments
|
||||
- `folder_path::String`
|
||||
Path to the folder containing text files.
|
||||
- `read_file_number::Integer=1`
|
||||
Which file to read (1-based index). Defaults to the first file.
|
||||
|
||||
# Example
|
||||
utils.read_textfile_by_index(cleaned_data_path, 2)
|
||||
read 2nd txt file in a folder
|
||||
# Return
|
||||
- A tuple of `(file_number::Integer, filename::String, content)` where:
|
||||
- `file_number` is the index of the file that was read
|
||||
- `filename` is the actual filename string
|
||||
- `content` is a vector of lines from the file (or `nothing` if error)
|
||||
|
||||
# Notes
|
||||
- Errors if `folder_path` is not a valid directory.
|
||||
- Errors if `read_file_number` exceeds the number of files in the folder.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> result = read_textfile_by_index("/path/to/folder", 2)
|
||||
(2, "sample.txt", ["line1", "line2", ...])
|
||||
```
|
||||
"""
|
||||
function read_textfile_by_index(folder_path::String, read_file_number::Integer=1)
|
||||
if isdir(folder_path)
|
||||
@@ -186,40 +203,39 @@ function read_textfile_by_index(folder_path::String, read_file_number::Integer=1
|
||||
end
|
||||
|
||||
|
||||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary.
|
||||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into an OrderedDict.
|
||||
|
||||
# What it does
|
||||
- Walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`,
|
||||
`Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where
|
||||
every dictionary-like node is a plain `Dict` and every array-like node is a
|
||||
`Vector{Any}`. Scalar values (numbers, strings, booleans, `nothing`, etc.)
|
||||
are returned unchanged.
|
||||
- Does **not** mutate the input; it always allocates new containers.
|
||||
The function walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`,
|
||||
`Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where
|
||||
every dictionary-like node is an `OrderedDict` and every array-like node is a `Vector{Any}`.
|
||||
Scalar values (numbers, strings, booleans, `nothing`, etc.) are returned unchanged.
|
||||
Does **not** mutate the input; it always allocates new containers.
|
||||
|
||||
# Arguments
|
||||
- `x`
|
||||
Any Julia value. If `x` is an `AbstractDict` it will be converted to a `Dict`;
|
||||
Any Julia value. If `x` is an `AbstractDict` it will be converted to an `OrderedDict`;
|
||||
if it is an `AbstractArray` its elements will be processed recursively.
|
||||
|
||||
# Keyword Arguments
|
||||
- `stringkey::Bool=false`
|
||||
If `true`, every dictionary key is converted to `String` via `string(k)`.
|
||||
If `false`, original key objects are preserved (useful when keys are already
|
||||
`String`, `Symbol`, or other types you want to keep).
|
||||
- `keytype::Type=Any`
|
||||
The key type for the output OrderedDict. Use `String` for `OrderedDict{String,Any}`,
|
||||
`Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types.
|
||||
- `sort_order::Union{Nothing, Vector}=nothing`
|
||||
Vector of keys specifying the desired order. Keys are arranged in the specified order
|
||||
first, followed by any remaining keys.
|
||||
|
||||
# Return
|
||||
- A newly allocated nested structure composed of `Dict{Any,Any}` and
|
||||
`Vector{Any}` that mirrors the input shape but uses plain Julia containers.
|
||||
- A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and `Vector{Any}`
|
||||
that mirrors the input shape but uses ordered Julia containers.
|
||||
|
||||
# Notes
|
||||
- The function treats any `AbstractDict` as a mapping source, so it works with
|
||||
`JSON.Object`, `Dict`, `OrderedDict`, etc.
|
||||
- Arrays are returned as `Vector{Any}` with their elements processed
|
||||
recursively.
|
||||
- Arrays are returned as `Vector{Any}` with their elements processed recursively.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using JSON
|
||||
julia> using JSON, DataStructures
|
||||
julia> d = Dict(
|
||||
"a" => 4,
|
||||
"b" => 6,
|
||||
@@ -234,92 +250,105 @@ julia> d = Dict(
|
||||
)
|
||||
)
|
||||
|
||||
julia jsonstring = JSON.json(d)
|
||||
julia> jsonstring = JSON.json(d)
|
||||
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
||||
julia> A2 = dictify(A1)
|
||||
Dict{Any,Any} with 3 entries:
|
||||
julia> A2 = dictify(A1; keytype=String)
|
||||
OrderedDict{String,Any} with 3 entries:
|
||||
"a" => 4
|
||||
"b" => 6
|
||||
"c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||
"c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||
|
||||
julia> A3 = dictify(A1; stringkey=false) # preserves original key objects
|
||||
julia> B1 = dictify(d; stringkey=true) # convert all keys in to string
|
||||
Dict{Any, Any} with 3 entries:
|
||||
"c" => Dict{Any, Any}("e"=>Dict{Any, Any}("f"=>"hey", "g"=>Dict{Any, Any}("world"=>Any[1, "2", 3, Dict{Any, Any}("dd"=>4.7)])), "d"=>7)
|
||||
"b" => 6
|
||||
"a" => 4
|
||||
julia> A3 = dictify(A1; keytype=Symbol)
|
||||
OrderedDict{Symbol,Any} with 3 entries:
|
||||
:a => 4
|
||||
:b => 6
|
||||
:c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||
|
||||
julia> B1 = dictify(d; keytype=String)
|
||||
OrderedDict{String, Any} with 3 entries:
|
||||
```
|
||||
|
||||
**With sort_order:**
|
||||
```jldoctest
|
||||
julia> d = Dict("a"=>1, "b"=>2, "c"=>3)
|
||||
julia> dictify(d; sort_order=["c", "a"])
|
||||
OrderedDict{String,Int} with 3 entries:
|
||||
"c" => 3
|
||||
"a" => 1
|
||||
"b" => 2
|
||||
```
|
||||
"""
|
||||
function dictify(x; stringkey::Bool=false)
|
||||
# Dict-like objects
|
||||
if x isa AbstractDict
|
||||
# choose output key type container (String keys when requested)
|
||||
out = Dict{Any,Any}()
|
||||
for (k,v) in x
|
||||
newk = stringkey ? string(k) : k
|
||||
out[newk] = dictify(v; stringkey=stringkey)
|
||||
end
|
||||
return out
|
||||
# Arrays / vectors: map elements recursively and return a Vector{Any}
|
||||
elseif x isa AbstractArray
|
||||
return [dictify(element; stringkey=stringkey) for element in x]
|
||||
# everything else: return as-is (primitives, numbers, strings, etc.)
|
||||
else
|
||||
return x
|
||||
end
|
||||
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||
)::OrderedDict where {T<:AbstractDict}
|
||||
# Dict-like objects
|
||||
out = OrderedDict{keytype, Any}()
|
||||
|
||||
# 1. Process and normalize all keys from the input dictionary
|
||||
processed_dict = OrderedDict{keytype, Any}()
|
||||
for (k, v) in x
|
||||
if keytype === String
|
||||
newk = string(k)
|
||||
elseif keytype === Symbol
|
||||
newk = Symbol(string(k))
|
||||
else
|
||||
newk = k
|
||||
end
|
||||
processed_dict[newk] = dictify(v; keytype=keytype, sort_order=sort_order)
|
||||
end
|
||||
|
||||
# 2. If a sort order is specified, apply it
|
||||
if !isnothing(sort_order)
|
||||
# Normalize the sort_order elements to match the requested keytype
|
||||
normalized_order = map(sort_order) do tk
|
||||
if keytype === String
|
||||
return string(tk)
|
||||
elseif keytype === Symbol
|
||||
return Symbol(string(tk))
|
||||
else
|
||||
return tk
|
||||
end
|
||||
end
|
||||
|
||||
# First, insert keys that match the requested order
|
||||
for target_key in normalized_order
|
||||
if haskey(processed_dict, target_key)
|
||||
out[target_key] = processed_dict[target_key]
|
||||
end
|
||||
end
|
||||
|
||||
# Then, append any remaining keys that weren't in the sort_order
|
||||
for (k, v) in processed_dict
|
||||
if !haskey(out, k)
|
||||
out[k] = v
|
||||
end
|
||||
end
|
||||
else
|
||||
# If no sort order is given, just use the processed dict
|
||||
out = processed_dict
|
||||
end
|
||||
|
||||
return out
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" Array_to_JSON_str(data::AbstractArray)
|
||||
|
||||
encode Array to JSON String
|
||||
|
||||
# Example
|
||||
|
||||
a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75]
|
||||
jsonStr = Array_to_JSON_str(a)
|
||||
jsonStr = "{\"Array\":[[1.23,9987.1,-0.0027],[4.7889,-123.07,-6.75]],\"size\":[3,2]}"
|
||||
"""
|
||||
function Array_to_JSON_str(data::AbstractArray)
|
||||
d = Dict("Array"=> data, "size"=>size(data))
|
||||
jsonStr = JSON.json(d)
|
||||
return jsonStr
|
||||
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||
) where {T<:AbstractArray}
|
||||
return [dictify(element; keytype=keytype, sort_order=sort_order) for element in x]
|
||||
end
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" JSON_str_to_Array(json_str::String)
|
||||
|
||||
decode JSON String to Array
|
||||
|
||||
# Example
|
||||
|
||||
jsonStr = "{\"Array\":[[1.23,9987.1,-0.0027],[4.7889,-123.07,-6.75]],\"size\":[3,2]}"
|
||||
a = JSON_str_to_Array(jsonStr)
|
||||
|
||||
"""
|
||||
function JSON_str_to_Array(jsonStr::String)
|
||||
jsonObj = JSON.parse(jsonStr)
|
||||
a = Array(jsonObj.Array)
|
||||
array = hcat(a...)
|
||||
return array
|
||||
function dictify(x; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||
)
|
||||
return x
|
||||
end
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary.
|
||||
|
||||
# What it does
|
||||
- Walks any nested structure composed of AbstractDict (e.g., JSON.Object,
|
||||
Dict, OrderedDict) and AbstractArray and produces a new tree where
|
||||
every dictionary-like node is an OrderedDict{Any,Any} and every array-like
|
||||
node is a Vector{Any}. Scalar values (numbers, strings, booleans,
|
||||
nothing, etc.) are returned unchanged.
|
||||
- Does **not** mutate the input; it always allocates new containers.
|
||||
The function walks any nested structure composed of AbstractDict (e.g., JSON.Object,
|
||||
Dict, OrderedDict) and AbstractArray and produces a new tree where
|
||||
every dictionary-like node is an OrderedDict{Any,Any} and every array-like
|
||||
node is a Vector{Any}. Scalar values (numbers, strings, booleans,
|
||||
nothing, etc.) are returned unchanged.
|
||||
Does **not** mutate the input; it always allocates new containers.
|
||||
|
||||
# Arguments
|
||||
- `x`
|
||||
@@ -328,13 +357,11 @@ end
|
||||
processed recursively.
|
||||
|
||||
# Keyword Arguments
|
||||
- `stringkey::Bool=false`
|
||||
If `true`, every dictionary key is converted to `String` via `string(k)`.
|
||||
If `false`, original key objects are preserved (useful when keys are already
|
||||
`String`, `Symbol`, or other types you want to keep).
|
||||
- `keytype::Type=Any`
|
||||
The key type for the output Dict. Use `String` for `OrderedDict{String,Any}`, `Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types.
|
||||
|
||||
# Return
|
||||
- A newly allocated nested structure composed of `OrderedDict{Any,Any}` and
|
||||
- A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and
|
||||
`Vector{Any}` that mirrors the input shape but uses ordered Julia containers.
|
||||
|
||||
# Notes
|
||||
@@ -361,41 +388,52 @@ julia> d = Dict(
|
||||
|
||||
julia jsonstring = JSON.json(d)
|
||||
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
||||
julia> A2 = OrderedDict(A1)
|
||||
Dict{Any,Any} with 3 entries:
|
||||
julia> A2 = ordereddictify(A1; keytype=String)
|
||||
OrderedDict{String,Any} with 3 entries:
|
||||
"a" => 4
|
||||
"b" => 6
|
||||
"c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||
"c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||
|
||||
julia> A3 = OrderedDict(A1; stringkey=false) # preserves original key objects
|
||||
julia> B1 = OrderedDict(d; stringkey=true) # convert all keys in to string
|
||||
Dict{Any, Any} with 3 entries:
|
||||
"c" => Dict{Any, Any}("e"=>Dict{Any, Any}("f"=>"hey", "g"=>Dict{Any, Any}("world"=>Any[1, "2", 3, Dict{Any, Any}("dd"=>4.7)])), "d"=>7)
|
||||
julia> A3 = ordereddictify(A1; keytype=Symbol)
|
||||
OrderedDict{Symbol,Any} with 3 entries:
|
||||
:a => 4
|
||||
:b => 6
|
||||
:c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||
|
||||
julia> B1 = ordereddictify(d; keytype=String)
|
||||
OrderedDict{String, Any} with 3 entries:
|
||||
"c" => OrderedDict{String, Any}("e"=>OrderedDict{String, Any}("f"=>"hey", "g"=>OrderedDict{String, Any}("world"=>Any[1, "2", 3, OrderedDict{String, Any}("dd"=>4.7)])), "d"=>7)
|
||||
"b" => 6
|
||||
"a" => 4
|
||||
```
|
||||
Ref. https://github.com/andyferris/Dictionaries.jl
|
||||
"""
|
||||
function ordereddictify(x; stringkey::Bool=false)
|
||||
function ordereddictify(x; keytype::Type=Any)
|
||||
# Dict-like objects
|
||||
if x isa AbstractDict
|
||||
# choose output key type container (String keys when requested)
|
||||
out = OrderedDict{Any,Any}()
|
||||
# choose output key type container
|
||||
out = OrderedDict{keytype,Any}()
|
||||
for (k,v) in x
|
||||
newk = stringkey ? string(k) : k
|
||||
out[newk] = ordereddictify(v; stringkey=stringkey)
|
||||
if keytype === String
|
||||
newk = string(k)
|
||||
elseif keytype === Symbol
|
||||
newk = Symbol(string(k))
|
||||
else
|
||||
newk = k
|
||||
end
|
||||
out[newk] = ordereddictify(v; keytype=keytype)
|
||||
end
|
||||
return out
|
||||
# Arrays / vectors: map elements recursively and return a Vector{Any}
|
||||
elseif x isa AbstractArray
|
||||
return [ordereddictify(element; stringkey=stringkey) for element in x]
|
||||
return [ordereddictify(element; keytype=keytype) for element in x]
|
||||
# everything else: return as-is (primitives, numbers, strings, etc.)
|
||||
else
|
||||
return x
|
||||
end
|
||||
end
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
#----------------------------------------------100---------------------------------------------
|
||||
|
||||
"""
|
||||
print time of cpu executtion at the line inwhich this macro is used
|
||||
@@ -477,6 +515,32 @@ function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector
|
||||
return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,))
|
||||
end
|
||||
|
||||
""" Compute the linearly weighted average of an array.
|
||||
The function assigns weights proportional to position indices (1, 2, 3, ...) to array
|
||||
elements and returns the weighted average.
|
||||
|
||||
# Arguments
|
||||
- `a::Array`
|
||||
Array of numeric values. Elements must support multiplication with numbers
|
||||
and summation.
|
||||
|
||||
# Return
|
||||
- The linearly weighted average as a floating-point number.
|
||||
|
||||
# Formula
|
||||
For an array `a` with `n` elements, computes:
|
||||
```
|
||||
sum(i * a[i]) / sum(a) for i = 1 to n
|
||||
```
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> a = [10, 20, 30]
|
||||
julia> linearly_weighted_avg(a)
|
||||
23.333333333333332
|
||||
```
|
||||
"""
|
||||
function linearly_weighted_avg(a::Array)
|
||||
total = 0.0
|
||||
for (i, v) in enumerate(a)
|
||||
@@ -487,10 +551,34 @@ function linearly_weighted_avg(a::Array)
|
||||
end
|
||||
|
||||
|
||||
""" Convert String that is holded inside a variable to Symbol
|
||||
# Example
|
||||
x = "hello" # x is a variable holding String "hello" \n
|
||||
y = variable_to_symbol(x) # y holds :hello
|
||||
""" Convert a variable's value (String) into a Symbol.
|
||||
The function takes a variable containing a String value and converts it to a Symbol
|
||||
using Julia's expression interpolation mechanism.
|
||||
|
||||
# Arguments
|
||||
- `variable`
|
||||
Any variable whose value is a String. The function uses `string(variable)`
|
||||
internally to obtain the value.
|
||||
|
||||
# Return
|
||||
- A `Symbol` constructed from the string value of the input variable.
|
||||
|
||||
# Notes
|
||||
- This function uses variable interpolation to capture the variable's
|
||||
value as a Symbol. It works with any variable type that can be converted
|
||||
to String.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> x = "hello"
|
||||
julia> variable_str_to_symbol(x)
|
||||
:hello
|
||||
|
||||
julia> y = "world_test"
|
||||
julia> variable_str_to_symbol(y)
|
||||
:world_test
|
||||
```
|
||||
"""
|
||||
function variable_str_to_symbol(variable)
|
||||
semi = :($variable)
|
||||
@@ -534,6 +622,43 @@ function fieldname_useable_type(somestruct, fieldname::Symbol;
|
||||
end
|
||||
|
||||
|
||||
""" Draw unique elements from a list without replacement.
|
||||
The function randomly selects a specified number of distinct elements from a collection,
|
||||
optionally excluding certain elements from consideration. Uses in-place
|
||||
shuffling for efficiency.
|
||||
|
||||
# Arguments
|
||||
- `drawOptions::Array`
|
||||
Collection of elements to draw from.
|
||||
- `draw_number::Integer`
|
||||
Number of unique elements to draw.
|
||||
|
||||
# Keyword Arguments
|
||||
- `exclude_list::Union{AbstractArray,Nothing}=nothing`
|
||||
Elements to exclude from the drawing pool. If `nothing`, no elements are
|
||||
excluded.
|
||||
|
||||
# Return
|
||||
- An array of `draw_number` unique elements drawn from `drawOptions`, excluding
|
||||
any elements in `exclude_list`.
|
||||
|
||||
# Notes
|
||||
- The function copies `drawOptions` and shuffles in-place, then pops elements
|
||||
sequentially to ensure uniqueness.
|
||||
- Errors if `draw_number` exceeds the number of available elements after
|
||||
exclusion.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> options = [1, 2, 3, 4, 5]
|
||||
julia> randomNoRepeat(options, 3)
|
||||
[3, 1, 5]
|
||||
|
||||
julia> randomNoRepeat(options, 2; exclude_list=[1, 5])
|
||||
[4, 2]
|
||||
```
|
||||
"""
|
||||
function randomNoRepeat(drawOptions::Array, draw_number::Integer;
|
||||
exclude_list::Union{AbstractArray,Nothing}=nothing)
|
||||
draw_option = copy(drawOptions)
|
||||
@@ -700,22 +825,46 @@ function selectRange(d::Dict{Symbol, <:AbstractVector}, range)
|
||||
return newDict
|
||||
end
|
||||
|
||||
""" Assign value to a given Dict by array of keys
|
||||
""" Recursively traverses a nested dictionary structure using a vector of keys
|
||||
and assigns a value to the final key. Creates intermediate dictionaries
|
||||
if they don't exist.
|
||||
|
||||
# Example
|
||||
d = Dict(
|
||||
:a1=> Dict(:c=> 5),
|
||||
:a2=> Dict(
|
||||
:k=> 10,
|
||||
:b=> Dict(
|
||||
:s=> "target",
|
||||
)
|
||||
)
|
||||
)
|
||||
index = [:a2, :b, :s] \n
|
||||
assignDict!(d, [:a2, :b, :s], "wow")
|
||||
# Arguments
|
||||
- `dict::Dict`
|
||||
The root dictionary to traverse and modify.
|
||||
- `accessArray::Array{Symbol}`
|
||||
A vector of symbols representing the key path to traverse.
|
||||
- `valueToAssign`
|
||||
The value to assign at the final key in the path.
|
||||
|
||||
return 1 if no target key in a given dict.
|
||||
# Return
|
||||
- `0` on success (value assigned)
|
||||
- `1` if the path cannot be traversed (missing intermediate keys)
|
||||
|
||||
# Notes
|
||||
- The function walks through each key in `accessArray` except the last one,
|
||||
expecting intermediate keys to already exist in the dictionary.
|
||||
- If any intermediate key is missing, the function returns `1` without
|
||||
modifying the dictionary.
|
||||
- The final key in `accessArray` receives the `valueToAssign`.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> d = Dict(
|
||||
:a1=> Dict(:c=> 5),
|
||||
:a2=> Dict(
|
||||
:k=> 10,
|
||||
:b=> Dict(
|
||||
:s=> "target",
|
||||
)
|
||||
)
|
||||
)
|
||||
julia> assignDict!(d, [:a2, :b, :s], "wow")
|
||||
0
|
||||
julia> d[:a2][:b][:s]
|
||||
"wow"
|
||||
```
|
||||
"""
|
||||
function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign)
|
||||
wd = nothing
|
||||
@@ -736,9 +885,39 @@ function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign)
|
||||
end
|
||||
end
|
||||
|
||||
""" convert hour(0-23), minute(0-59) into julia time object
|
||||
# Example
|
||||
time
|
||||
""" Converts hour (0-23) and minute (0-59) into a Julia `Time` object using
|
||||
12-hour format with AM/PM indicator.
|
||||
|
||||
# Arguments
|
||||
- `h::Integer`
|
||||
Hour in 24-hour format (0 to 23).
|
||||
- `m::Integer`
|
||||
Minute (0 to 59).
|
||||
|
||||
# Return
|
||||
- A `Time` object representing the time in 12-hour format with AM/PM.
|
||||
|
||||
# Notes
|
||||
- Hours 0 and 12 are special cases: 0 becomes 12 AM, 12 becomes 12 PM.
|
||||
- Hours 1-11 remain the same with "am" suffix.
|
||||
- Hours 13-23 are converted to 1-11 with "pm" suffix.
|
||||
- Minutes less than 10 are zero-padded.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> iTime(0, 30)
|
||||
12:30 AM
|
||||
|
||||
julia> iTime(9, 15)
|
||||
9:15 AM
|
||||
|
||||
julia> iTime(12, 0)
|
||||
12:00 PM
|
||||
|
||||
julia> iTime(14, 5)
|
||||
2:05 PM
|
||||
```
|
||||
"""
|
||||
function iTime(h::Integer, m::Integer)
|
||||
if h == 0
|
||||
@@ -782,19 +961,37 @@ function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair)
|
||||
end
|
||||
|
||||
|
||||
""" Assign matrix b to matrix a according to matrix b's CartesianIndex.
|
||||
""" Assigns elements from matrix `b` to matrix `a` using the Cartesian indices
|
||||
of `b`. Elements are copied in the order they appear when iterating over `b`,
|
||||
and placed into `a` at the corresponding Cartesian positions of `b`.
|
||||
|
||||
Arguments:\n
|
||||
a : target matrix.
|
||||
b : source matrix.
|
||||
# Arguments
|
||||
- `a`
|
||||
Target matrix where values from `b` will be assigned.
|
||||
- `b`
|
||||
Source matrix whose Cartesian indices determine where values are placed in `a`.
|
||||
|
||||
Return:\n
|
||||
Resulting matrix a.
|
||||
# Return
|
||||
- `nothing`
|
||||
|
||||
Example:\n
|
||||
```jldoctest
|
||||
julia> not done yet
|
||||
```
|
||||
# Notes
|
||||
- The function iterates through `b` in column-major order (Julia's default),
|
||||
retrieving each element's Cartesian index and assigning it to the same
|
||||
position in `a`.
|
||||
- Matrix `a` must have sufficient size to accommodate all Cartesian indices
|
||||
from `b`; otherwise, an `BoundsError` may occur.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> a = zeros(4, 4);
|
||||
julia> b = [1 2; 3 4];
|
||||
julia> cartesianAssign!(a, b);
|
||||
julia> a[1:2, 1:2]
|
||||
2×2 Matrix{Float64}:
|
||||
1.0 3.0
|
||||
2.0 4.0
|
||||
```
|
||||
"""
|
||||
function cartesianAssign!(a, b)
|
||||
for (i, v) in enumerate(b)
|
||||
@@ -1149,7 +1346,6 @@ julia> text = "{\"ask\": {\"text\": \"Could you please tell me about the special
|
||||
julia> GeneralUtils.getStringBetweenCharacters(text, '{', '}', endCharLocation="end")
|
||||
"{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}"
|
||||
```
|
||||
# Signature
|
||||
"""
|
||||
function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char;
|
||||
endCharLocation::String="next", includeChar::Bool=true)::String where {T<:AbstractString}
|
||||
@@ -1181,59 +1377,40 @@ function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char;
|
||||
end
|
||||
|
||||
|
||||
|
||||
""" Read JSON string and return a dictionary with string key. (JSON3 defaults to symbol key)
|
||||
This function solve the problem of reading JSON with string key.
|
||||
|
||||
Arguments:
|
||||
jsonString::String
|
||||
|
||||
Return:
|
||||
a dictionary with string key
|
||||
|
||||
Example:
|
||||
```jldoctest
|
||||
julia> jsonString = {\"wine type\": \"Red\", \"intensity level\": \"medium-bodied\"}
|
||||
julia> JSON3read_stringKey(jsonString)
|
||||
Dict{String, Any} with 2 entries:
|
||||
"intensity level" => "medium-bodied"
|
||||
"wine type" => "Red"
|
||||
```
|
||||
"""
|
||||
function JSON3read_stringKey(jsonString::AbstractString)
|
||||
jsonobj = JSON3.read(jsonString)
|
||||
newDict = OrderedDict{String,Any}()
|
||||
for (k,v) in jsonobj
|
||||
newDict[string(k)] = v
|
||||
end
|
||||
return newDict
|
||||
end
|
||||
|
||||
|
||||
""" Create nested dict path if it does not already exist. The same concept as Julia's mkpath()
|
||||
""" Recursively creates nested dictionary paths if they do not exist and assigns
|
||||
a value to the final key. Similar to `mkpath()` but for dictionaries.
|
||||
|
||||
# Arguments
|
||||
- `dict::Dict`
|
||||
target dict
|
||||
- `addkeys::Union{Vector{String}, Vector{Symbol}}`
|
||||
keys to be added to dict
|
||||
- `value`
|
||||
value to be added to dict at final key in keypath
|
||||
- `dict::Union{Dict{Symbol, Any}, Dict{String, Any}}`
|
||||
The target dictionary to traverse and modify. Must use consistent key types
|
||||
(either all `String` or all `Symbol`).
|
||||
- `addkeys::Union{Vector{String}, Vector{Symbol}}`
|
||||
A vector of keys representing the path to traverse. Intermediate dictionaries
|
||||
are created if they don't exist.
|
||||
- `value`
|
||||
The value to assign at the final key in the path.
|
||||
|
||||
# Return
|
||||
- dict with added keypath
|
||||
- The assigned `value`.
|
||||
|
||||
# Example
|
||||
# Notes
|
||||
- The function ensures key type consistency: the type of keys being added must
|
||||
match the type of existing keys in the dictionary.
|
||||
- Intermediate dictionaries are automatically created with the appropriate key
|
||||
type when they don't exist.
|
||||
- The function walks through each key in `addkeys` except the last one,
|
||||
creating intermediate dictionaries as needed, and assigns `value` to the final
|
||||
key.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using GeneralUtils
|
||||
julia> d = Dict{String, Any}("a" => Dict{String, Any}("b" => 10))
|
||||
julia> GeneralUtils.mkDictPath!(d, ["a", "v", "x", "y", "z"], 42)
|
||||
Dict{String, Any} with 1 entry:
|
||||
"path" => Dict{Any, Any}("to"=>Dict{Any, Any}("nested"=>Dict{Any, Any}("value"=>42)))
|
||||
julia> d = Dict("a" => Dict("b" => 10))
|
||||
julia> mkDictPath!(d, ["a", "v", "x", "y", "z"], 42)
|
||||
42
|
||||
julia> d["a"]["v"]["x"]["y"]["z"]
|
||||
42
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}},
|
||||
addkeys::Union{Vector{String}, Vector{Symbol}}, value)
|
||||
@@ -1257,27 +1434,33 @@ function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}},
|
||||
end
|
||||
|
||||
|
||||
""" Get nested dict value using a vector of keys
|
||||
""" Retrieves a value from a nested dictionary by traversing a vector of keys.
|
||||
Creates intermediate dictionaries if they don't exist.
|
||||
|
||||
# Arguments
|
||||
- `dict::Dict`
|
||||
target dict
|
||||
- `keys::Vector`
|
||||
keys vector
|
||||
- `dict::Dict`
|
||||
The root dictionary to traverse.
|
||||
- `keys::Vector`
|
||||
A vector of keys representing the path to traverse. Each key in the vector
|
||||
is used to access the next level of nesting.
|
||||
|
||||
# Return
|
||||
- dict with added keypath
|
||||
- The value at the final key in the path.
|
||||
|
||||
# Example
|
||||
# Notes
|
||||
- Errors with `ArgumentError` if any intermediate key is missing from the
|
||||
dictionary path.
|
||||
- The function walks through each key in `keys` except the last one,
|
||||
expecting intermediate keys to exist in the dictionary.
|
||||
- The final key in `keys` is used to retrieve the value.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using GeneralUtils
|
||||
julia> d = Dict{Symbol, Any}(:a => Dict{Symbol, Any}(:b => 10))
|
||||
julia> GeneralUtils.getDictPath(d, [:a, :b])
|
||||
julia> d = Dict(:a => Dict(:b => 10))
|
||||
julia> getDictPath(d, [:a, :b])
|
||||
10
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function getDictPath(dict::Dict, keys::Vector)
|
||||
current_dict = dict
|
||||
@@ -1429,8 +1612,6 @@ OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do",
|
||||
:plan => "wake up and going out",
|
||||
:action => "1. wake up 2. eat 3. sleep")
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function textToDict(text::String, detectKeywords::Vector{String};
|
||||
dictKey::Union{Vector{String}, Nothing}=nothing,
|
||||
@@ -1491,6 +1672,82 @@ function textToDict(text::String, detectKeywords::Vector{String};
|
||||
return od2
|
||||
end
|
||||
|
||||
""" Recursively convert dictionary into an HTML string representation.
|
||||
The function walks a nested `AbstractDict` structure and produces a well-formed HTML string
|
||||
where each dictionary key becomes an HTML tag. Nested dictionaries become
|
||||
nested tags, and scalar values (numbers, strings, etc.) become the text
|
||||
content of leaf tags.
|
||||
|
||||
# Arguments
|
||||
- `d::AbstractDict`
|
||||
The dictionary to convert. Keys must be strings or symbols that form valid
|
||||
HTML tag names.
|
||||
|
||||
# Keyword Arguments
|
||||
- `indent_level::Integer=1`
|
||||
Initial indentation level for the output. Each recursive level increases
|
||||
indentation by one.
|
||||
- `indent_str::String=" "`
|
||||
String used for each indentation level.
|
||||
|
||||
# Return
|
||||
- A single HTML string representing the dictionary structure, with proper
|
||||
opening and closing tags and appropriate indentation.
|
||||
|
||||
# Notes
|
||||
- Keys are sorted alphabetically for deterministic output.
|
||||
- Works recursively: dictionary values produce nested tags; non-dictionary
|
||||
values are placed as text between opening/closing tags.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> d = Dict(
|
||||
"html" => Dict(
|
||||
"head" => Dict("title" => "Test"),
|
||||
"body" => Dict(
|
||||
"h1" => "Hello",
|
||||
"p" => "World"
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
julia> println(dict_to_string_html(d))
|
||||
<html>
|
||||
<head>
|
||||
<title>Test</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hello</h1>
|
||||
<p>World</p>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
"""
|
||||
function dict_to_string_html(d::AbstractDict; indent_level=1, indent_str=" ")
|
||||
lines = String[]
|
||||
padding = indent_str ^ indent_level
|
||||
|
||||
for k in keys(d)
|
||||
v = d[k]
|
||||
|
||||
if v isa AbstractDict
|
||||
# Open tag, recurse for children, then close tag
|
||||
push!(lines, "$padding<$k>")
|
||||
ind_level = indent_level + 1
|
||||
push!(lines, dict_to_string_html(v; indent_level=ind_level, indent_str=indent_str))
|
||||
push!(lines, "$padding</$k>")
|
||||
else
|
||||
# Leaf node: put key and value on a single line
|
||||
push!(lines, "$padding<$k>$v</$k>")
|
||||
end
|
||||
end
|
||||
return join(lines, "\n")
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+1002
-335
File diff suppressed because it is too large
Load Diff
+140
-85
@@ -5,7 +5,7 @@ export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, rep
|
||||
dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString,
|
||||
dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
|
||||
dictToString_numbering, extract_triple_backtick_text,
|
||||
countGivenWords, remove_french_accents,
|
||||
countGivenWords, remove_french_accents, removestring,
|
||||
extractTextBetweenCharacter, extractTextBetweenString,
|
||||
convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
|
||||
|
||||
@@ -13,31 +13,34 @@ using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" Compute time different between start time and stop time in a given unit.
|
||||
Unit can be "milliseconds", "seconds", "minutes", "hours".
|
||||
""" Computes the time difference between two `DateTime` values and returns the
|
||||
result in a specified unit: milliseconds, seconds, minutes, or hours.
|
||||
|
||||
# Arguments
|
||||
- `starttime::DateTime`
|
||||
start time
|
||||
- `stoptime::DateTime`
|
||||
stop time
|
||||
- `unit::String`
|
||||
unit of time difference
|
||||
- `starttime::DateTime`
|
||||
The starting `DateTime` value.
|
||||
- `stoptime::DateTime`
|
||||
The ending `DateTime` value.
|
||||
- `unit::String`
|
||||
The unit for the result. Must be one of: `"milliseconds"`, `"seconds"`,
|
||||
`"minutes"`, `"hours"`.
|
||||
|
||||
# Return
|
||||
- time difference in given unit
|
||||
- `Integer`: The time difference converted to the specified unit.
|
||||
|
||||
# Example
|
||||
# Notes
|
||||
- The function computes `stoptime - starttime` and converts the result to the
|
||||
requested unit using integer division.
|
||||
- Errors with `ArgumentError` if an invalid unit is specified.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using GeneralUtils, Dates
|
||||
julia> a = Dates.now()
|
||||
julia> b = a + Dates.Day(5) # add 5 days
|
||||
julia> GeneralUtils.timedifference(a, b, "hours")
|
||||
julia> b = a + Dates.Day(5)
|
||||
julia> timedifference(a, b, "hours")
|
||||
120
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer
|
||||
diff = stoptime - starttime
|
||||
@@ -184,21 +187,27 @@ end
|
||||
|
||||
|
||||
|
||||
""" Get uuid4 with snake case
|
||||
""" Generates a UUID4 (version 4) identifier and converts it to snake case by
|
||||
replacing hyphens with underscores.
|
||||
|
||||
# Arguments
|
||||
- This function takes no arguments.
|
||||
|
||||
# Return
|
||||
- `uuid4::String`
|
||||
uuid4 with snake case
|
||||
- `String`: A UUID4 string with underscores instead of hyphens (e.g.,
|
||||
`"0f6e4f_568c_4df4_8c79_1d7a58072f4a"`).
|
||||
|
||||
# Example
|
||||
# Notes
|
||||
- Uses the `uuid4()` function from the UUIDs standard library to generate a
|
||||
random UUID.
|
||||
- The underscore character replaces all hyphens in the UUID string.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using GeneralUtils
|
||||
julia> GeneralUtils.uuid4snakecase()
|
||||
julia> uuid4snakecase()
|
||||
"0f6e4f_568c_4df4_8c79_1d7a58072f4a"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function uuid4snakecase()::String
|
||||
_id = string(uuid4())
|
||||
@@ -207,32 +216,37 @@ function uuid4snakecase()::String
|
||||
end
|
||||
|
||||
|
||||
""" Replace a dictionary key with the new key
|
||||
""" Replaces keys in a dictionary according to a mapping, returning a new
|
||||
dictionary with updated keys while preserving the original values.
|
||||
|
||||
# Arguments
|
||||
- `d::Dict`
|
||||
The input dictionary that you want to modify
|
||||
- `replacementMap::Dict`
|
||||
A dictionary that maps old keys to new keys
|
||||
- `d::Dict`
|
||||
The input dictionary to modify.
|
||||
- `replacementMap::Dict`
|
||||
A dictionary mapping old keys to new keys. Keys not present in this map are
|
||||
left unchanged.
|
||||
|
||||
# Return
|
||||
- `newDict::Dict`
|
||||
new dictionary with the replaced keys
|
||||
- `Dict`: A new dictionary with replaced keys. Values are preserved from the
|
||||
original dictionary.
|
||||
|
||||
# Example
|
||||
# Notes
|
||||
- The function creates a new dictionary rather than modifying the input in
|
||||
place.
|
||||
- Keys not found in `replacementMap` are copied to the result with their
|
||||
original keys unchanged.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using GeneralUtils
|
||||
julia> d = Dict(:a => 1, :b => 2, :c => 3)
|
||||
julia> replacement_map = Dict(:a => :x, :b => :y)
|
||||
julia> new_dict = GeneralUtils.replaceDictKeys(d, replacement_map)
|
||||
julia> replaceDictKeys(d, replacement_map)
|
||||
Dict{Any, Any} with 3 entries:
|
||||
:y => 2
|
||||
:c => 3
|
||||
:x => 1
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
|
||||
newDict = Dict()
|
||||
@@ -294,36 +308,43 @@ end
|
||||
|
||||
|
||||
|
||||
""" Execute a function with timer.
|
||||
""" Executes a function with a timeout mechanism. If the function does not
|
||||
complete within the specified time, it is interrupted and a timeout message
|
||||
is returned.
|
||||
|
||||
# Arguments
|
||||
- `f::Function`
|
||||
a function to run
|
||||
- `timeoutwindow::Integer``
|
||||
timeout in seconds
|
||||
- `f::Function`
|
||||
The function to execute.
|
||||
- `timeoutwindow::Integer`
|
||||
The timeout duration in seconds.
|
||||
|
||||
# Keyword Argument
|
||||
- `fargs`
|
||||
arguments for the function
|
||||
- `timeoutmsg::String`
|
||||
time out message
|
||||
# Keyword Arguments
|
||||
- `fargs`
|
||||
Arguments to pass to the function `f`. If `nothing`, the function is called
|
||||
without arguments.
|
||||
- `timeoutmsg::String`
|
||||
The message to return if the function times out. Defaults to `"task timed out"`.
|
||||
|
||||
# Return
|
||||
- task result otherwise timeout message
|
||||
- The result of the function if it completes within the timeout, otherwise the
|
||||
`timeoutmsg` string.
|
||||
|
||||
# Example
|
||||
```jldoctest
|
||||
# Notes
|
||||
- Uses Julia's `@task`, `schedule`, and `Timer` to implement non-blocking
|
||||
execution with interruption via `Base.throwto`.
|
||||
- Errors with `InterruptException` if the function exceeds the timeout.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> function testfunc(x)
|
||||
sleep(x)
|
||||
return "task done"
|
||||
end
|
||||
sleep(x)
|
||||
return "task done"
|
||||
end
|
||||
julia> result = timeout(testfunc, 10; fargs=20)
|
||||
"task timed out"
|
||||
julia> result = timeout(testfunc, 20; fargs=10)
|
||||
"task done"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function timeout(f::Function, timeoutwindow::Integer; fargs=nothing, timeoutmsg="task timed out")
|
||||
tsk = @task f(fargs)
|
||||
@@ -340,23 +361,26 @@ end
|
||||
|
||||
|
||||
|
||||
""" Convert a dataframe into CSV.
|
||||
""" Converts a DataFrame to a CSV string representation using the CSV.jl package.
|
||||
|
||||
# Arguments
|
||||
- `df::DataFrame`
|
||||
A connection object to Postgres database
|
||||
- `df::DataFrame`
|
||||
The DataFrame to convert to CSV format.
|
||||
|
||||
# Return
|
||||
- `result::String`
|
||||
- `String`: The DataFrame contents as a CSV-formatted string.
|
||||
|
||||
# Example
|
||||
# Notes
|
||||
- Uses `CSV.write` with an `IOBuffer` to capture the output as a string.
|
||||
- The returned string contains the full CSV representation including headers.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using DataFrames, GeneralUtils
|
||||
julia> df = DataFrame(A=1:3, B=5:7, fixed=1)
|
||||
julia> result = GeneralUtils.dataframeToCSV(df)
|
||||
julia> dataframeToCSV(df)
|
||||
"1,5,1\n2,6,1\n3,7,1\n"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function dataframeToCSV(df::DataFrame)
|
||||
# Create an IOBuffer to capture the output
|
||||
@@ -426,8 +450,6 @@ end
|
||||
3 => [Dict("a"=>7), Dict("a"=>8), Dict("a"=>9)]
|
||||
4 => [Dict("a"=>10)]
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function disintegrate_vectorDict(data::Vector, partsize::Integer
|
||||
)
|
||||
@@ -473,8 +495,6 @@ end
|
||||
julia> getDataFrameValue(df[1, :], :name)
|
||||
"Alice"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
getDataFrameValue(row::DataFrameRow, key::Symbol) = row.:($key)
|
||||
|
||||
@@ -543,8 +563,6 @@ end
|
||||
julia> dfToString(df)
|
||||
"1) name: Alice, age: 25\n2) name: Bob, age: 30"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function dfToString(df::DataFrame)
|
||||
dfstr = ""
|
||||
@@ -583,8 +601,6 @@ end
|
||||
"{\"name\":\"Alice\",\"age\":25}"
|
||||
"{\"name\":\"Bob\",\"age\":30}"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function dataframe_to_json_list(df::DataFrame)::Vector{String}
|
||||
json_list = []
|
||||
@@ -618,8 +634,6 @@ end
|
||||
julia> dict_to_string(od)
|
||||
"1) name: Alice, 2) age: 25"
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function dictToString(od::T) where {T<:AbstractDict}
|
||||
items = []
|
||||
@@ -733,27 +747,39 @@ end
|
||||
|
||||
|
||||
|
||||
"""
|
||||
remove_french_accents(text::String) -> String
|
||||
""" Remove French accents from the given text.
|
||||
|
||||
Remove French accents from the given text.
|
||||
The function replaces accented French characters with their non-accented
|
||||
counterparts using a dictionary mapping. Supported accented characters
|
||||
include: à, â, ä, á, é, è, ê, ë, î, ï, í, ñ, ô, ö, ò, ó, ù, û, ü, ÿ, ç,
|
||||
and their uppercase variants. The apostrophe character `’` is removed
|
||||
completely.
|
||||
|
||||
# Arguments
|
||||
- `text::String` The input string containing French accents.
|
||||
- `text::AbstractString`
|
||||
The input string containing French accented characters.
|
||||
|
||||
# Returns
|
||||
- `String` The input string with all French accents removed.
|
||||
# Return
|
||||
- `AbstractString`: A new string with all French accents replaced by their
|
||||
non-accented equivalents.
|
||||
|
||||
# Notes
|
||||
- The function creates a character list and replaces each accented character
|
||||
according to an internal dictionary mapping.
|
||||
- Does **not** mutate the input; it allocates a new string.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> remove_french_accents("Café")
|
||||
"Cafe"
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> remove_french_accents("Café")
|
||||
"Cafe"
|
||||
|
||||
julia> remove_french_accents("L'été est beau.")
|
||||
"L'ete est beau."
|
||||
```
|
||||
julia> remove_french_accents("L'été est beau.")
|
||||
"L'ete est beau."
|
||||
|
||||
# Signature
|
||||
julia> remove_french_accents("Noël, naïve, François")
|
||||
"Noel, naive, Francois"
|
||||
```
|
||||
"""
|
||||
function remove_french_accents(text::AbstractString)::AbstractString
|
||||
textcharlist = [i for i in text]
|
||||
@@ -1113,9 +1139,38 @@ function nonRecentElementsIndex(vectorlength::Integer, n::Integer)
|
||||
return 1:(vectorlength-n)
|
||||
end
|
||||
|
||||
""" Remove specified substrings from text.
|
||||
|
||||
Removes all occurrences of each string in `removelist` from the input text
|
||||
by repeatedly replacing them with empty strings until none remain.
|
||||
|
||||
# Arguments
|
||||
- `text::String`
|
||||
The input string to modify.
|
||||
- `removelist::Vector{String}`
|
||||
A vector of substrings to remove from the text.
|
||||
|
||||
# Return
|
||||
- `String`: The text with all specified substrings removed.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> using GeneralUtils
|
||||
julia> removestring("hello world", ["l", " "])
|
||||
"heoword"
|
||||
julia> removestring("foo bar baz", ["bar", " "])
|
||||
"foobaz"
|
||||
```
|
||||
"""
|
||||
function removestring(text::String, removelist::Vector{String})::String
|
||||
for i in removelist
|
||||
while occursin(i, text)
|
||||
text = replace(text, i => "")
|
||||
end
|
||||
end
|
||||
|
||||
return string(text)
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user