Compare commits
70 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 517a08365d | |||
| d032cec0b2 | |||
| 64575d81b5 | |||
| fb54fddf10 | |||
| 407447831a | |||
| c475eb169c | |||
| d1a279cca2 | |||
| bf3b65ee7b | |||
| 360d64c474 | |||
| 41a354fa73 | |||
| 801596fa7f | |||
| 2fbe9d6e1a | |||
| 2c2690e5dd | |||
| 95db5f877d | |||
| 7e2ddd846e | |||
| ab113acde5 | |||
| 7391f0f2ce | |||
| 01f4e52c64 | |||
| 35acfe5b70 | |||
| e1ebbf370e | |||
| 1fd3fa1bee | |||
| 9721a393bc | |||
| 1db8e4e383 | |||
| c51dfc549c | |||
| f0ad6a3e48 | |||
| 07d5d0f885 | |||
| c829bf65f6 | |||
| b09efc9068 | |||
| 6fb2d5f82b | |||
| 55cc0f78c9 | |||
| 73ec3bbb04 | |||
| 1ad46c6e18 | |||
| 95954249ce | |||
| a15630619a | |||
| f28405f3f1 | |||
| d658d9a25b | |||
| edad442242 | |||
| c56fc7366c | |||
| c4eeb99aba | |||
| bd5022c8bc | |||
| db84b1c398 | |||
| f8f8410259 | |||
| 956adf0b93 | |||
| 0f2a33bcdd | |||
| b8f84846bb | |||
| adf6264061 | |||
| 08f19f17a2 | |||
| 8a4e882dc1 | |||
| fb7942a965 | |||
| e54454b099 | |||
| dac98ab38c | |||
| 1e8149aa6f | |||
| 8f12c29a78 | |||
| e3d09e6ebd | |||
| f33f4f0790 | |||
| fcf2044dd9 | |||
| 05d8cb9c02 | |||
| 13de2f90ff | |||
| 22fe810f63 | |||
| abdf6cf3b8 | |||
| f2ba243df0 | |||
| bb2851332a | |||
| 00225f3a06 | |||
| 7cb0bd077f | |||
| 0ba2aa310e | |||
| 1916668c6e | |||
| 919800da42 | |||
| 947580a2ec | |||
| 688b9a22b6 | |||
| 76ce0fc54f |
+543
-112
File diff suppressed because it is too large
Load Diff
+14
-2
@@ -1,23 +1,35 @@
|
|||||||
name = "GeneralUtils"
|
name = "GeneralUtils"
|
||||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||||
version = "0.3.2"
|
version = "0.6.1"
|
||||||
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
||||||
|
|
||||||
[deps]
|
[deps]
|
||||||
|
AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
|
||||||
|
AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
|
||||||
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
|
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
|
||||||
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
||||||
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
||||||
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
||||||
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
|
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
|
||||||
|
Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
|
||||||
|
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||||
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
||||||
|
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
|
||||||
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
||||||
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
||||||
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||||
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
||||||
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
||||||
|
StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
|
||||||
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||||
|
|
||||||
[compat]
|
[compat]
|
||||||
JSON = "1.3.0"
|
AWS = "1.99.0"
|
||||||
|
AWSS3 = "0.11.6"
|
||||||
|
Graphs = "1.14.0"
|
||||||
|
HTTP = "2.5.0 - 2.9.9"
|
||||||
|
JSON = "1.3.0 - 1.9.9"
|
||||||
|
LibPQ = "1.18.0"
|
||||||
NATS = "0.1.0"
|
NATS = "0.1.0"
|
||||||
Revise = "3.13.2"
|
Revise = "3.13.2"
|
||||||
|
StringDistances = "1.0.0"
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ using .llmUtil
|
|||||||
include("interface.jl")
|
include("interface.jl")
|
||||||
using .interface
|
using .interface
|
||||||
|
|
||||||
|
include("garageS3.jl")
|
||||||
|
using .garageS3
|
||||||
|
|
||||||
#------------------------------------------------------------------------------------------------100
|
#------------------------------------------------------------------------------------------------100
|
||||||
|
|
||||||
|
|||||||
+139
-3
@@ -1,13 +1,107 @@
|
|||||||
module dbUtil
|
module dbUtil
|
||||||
|
|
||||||
export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
|
export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
|
||||||
|
execute_postgres_sql
|
||||||
|
|
||||||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
|
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
|
||||||
SHA
|
SHA, NATS, LibPQ
|
||||||
using ..util
|
using ..util
|
||||||
#[PENDING] update code to use JSON
|
#[PENDING] update code to use JSON
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
""" Execute SQL against a PostgreSQL database using LibPQ connection string.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
|
||||||
|
- `sql::AbstractString`: SQL query to execute
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- `LibPQ.Result` on success, `nothing` on failure
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
|
||||||
|
sql = "SELECT * FROM wine;"
|
||||||
|
result = execute_postgres_sql(pg_conn_str, sql)
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
|
||||||
|
db_connection = LibPQ.Connection(pg_conn_str)
|
||||||
|
result = nothing
|
||||||
|
try
|
||||||
|
result = LibPQ.execute(db_connection, sql)
|
||||||
|
catch e
|
||||||
|
@error e
|
||||||
|
LibPQ.close(db_connection)
|
||||||
|
end
|
||||||
|
|
||||||
|
LibPQ.close(db_connection)
|
||||||
|
return result
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" find_text_vector_similarity
|
||||||
|
|
||||||
|
Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
|
||||||
|
|
||||||
|
This function computes an embedding for the input text using the provided embedding function,
|
||||||
|
then queries the database to find records with the most similar vector representations using
|
||||||
|
PostgreSQL's cosine similarity operator (`<->`).
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `text::AbstractString`: The input text to find similar records for
|
||||||
|
- `tablename::AbstractString`: Name of the database table containing the embedding column
|
||||||
|
- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
|
||||||
|
- `executesql::Function`: Function that executes SQL queries and returns results
|
||||||
|
- `get_embedding::Function`: Function that generates embeddings for text inputs
|
||||||
|
|
||||||
|
# Keyword Arguments
|
||||||
|
- `limit::Integer=1`: Maximum number of similar records to return
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
|
||||||
|
where smaller values indicate higher similarity
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
# Assume you have embedding and SQL execution functions
|
||||||
|
text = "a rich structured red wine"
|
||||||
|
tablename = "wine"
|
||||||
|
embeddingColumnName = "description_embedding"
|
||||||
|
|
||||||
|
df = find_text_vector_similarity(
|
||||||
|
text, tablename, embeddingColumnName,
|
||||||
|
executesql, get_embedding;
|
||||||
|
limit = 5
|
||||||
|
)
|
||||||
|
|
||||||
|
# Result contains columns from the table plus a 'distance' column
|
||||||
|
# where distance = 1 - cosine_similarity (smaller = more similar)
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
|
||||||
|
executesql::Function, get_embedding;
|
||||||
|
limit::Integer=1
|
||||||
|
)::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
|
||||||
|
# get embedding from LLM service
|
||||||
|
_embedding = get_embedding([text])
|
||||||
|
_embedding = _embedding["data"][1]["embedding"]
|
||||||
|
_embedding = "$_embedding"
|
||||||
|
|
||||||
|
embedding = _embedding[4:end] # remove 'Any' from Any[...]
|
||||||
|
|
||||||
|
# check whether there is close enough vector already store in executesql. if no, add, else skip
|
||||||
|
sql = """
|
||||||
|
SELECT *, $embeddingColumnName <-> '$embedding' as distance
|
||||||
|
FROM $tablename
|
||||||
|
ORDER BY distance LIMIT $limit;
|
||||||
|
"""
|
||||||
|
response = executesql(sql)
|
||||||
|
df = DataFrame(response)
|
||||||
|
|
||||||
|
return df
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
|
dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
|
||||||
@@ -121,7 +215,8 @@ function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, d
|
|||||||
for (key, value) in data
|
for (key, value) in data
|
||||||
if key ∈ columnToInsert
|
if key ∈ columnToInsert
|
||||||
push!(columns, string(key))
|
push!(columns, string(key))
|
||||||
push!(values, "'$value'") #[] number should not wrapped in ''
|
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||||||
|
push!(values, value_str)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -131,6 +226,22 @@ function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, d
|
|||||||
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
function generateInsertSQL(table_name::String, data::AbstractDict{String, Any})
|
||||||
|
columns = String[]
|
||||||
|
values = String[]
|
||||||
|
|
||||||
|
for (key, value) in data
|
||||||
|
push!(columns, string(key))
|
||||||
|
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||||||
|
push!(values, value_str)
|
||||||
|
end
|
||||||
|
|
||||||
|
columns_str = join(columns, ", ")
|
||||||
|
values_str = join(values, ", ")
|
||||||
|
|
||||||
|
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
@@ -185,5 +296,30 @@ function generateUpdateSQL(table_name::String, pk_column::String, pk_value,
|
|||||||
return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
|
return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any},
|
||||||
|
data::AbstractDict{String, Any})
|
||||||
|
# Build SET clause
|
||||||
|
set_parts = String[]
|
||||||
|
for (key, value) in data
|
||||||
|
if key ∉ keys(pk_dict)
|
||||||
|
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||||||
|
push!(set_parts, "$(string(key)) = $value_str")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
set_clause = join(set_parts, ", ")
|
||||||
|
|
||||||
|
# Build WHERE clause for composite keys
|
||||||
|
where_parts = String[]
|
||||||
|
for (col, val) in pk_dict
|
||||||
|
val_str = isa(val, AbstractString) ? "'$val'" : "$val"
|
||||||
|
push!(where_parts, "$(string(col)) = $val_str")
|
||||||
|
end
|
||||||
|
|
||||||
|
where_clause = join(where_parts, " AND ")
|
||||||
|
|
||||||
|
return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
end # module
|
end # module
|
||||||
+254
@@ -0,0 +1,254 @@
|
|||||||
|
module garageS3
|
||||||
|
|
||||||
|
export
|
||||||
|
GarageStorage,
|
||||||
|
put_file,
|
||||||
|
get_file,
|
||||||
|
list_files,
|
||||||
|
delete_file
|
||||||
|
|
||||||
|
using AWS, AWSS3
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
||||||
|
# this file use local AWS config
|
||||||
|
|
||||||
|
""" Example
|
||||||
|
const storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
||||||
|
"sommpanion-s3"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
||||||
|
put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
|
||||||
|
keys = list_files(storage)
|
||||||
|
data = String(get_file(storage, "users-1004.json"))
|
||||||
|
|
||||||
|
println("Read back: ", data)
|
||||||
|
println("Bucket keys: ", keys)
|
||||||
|
|
||||||
|
# test with curl
|
||||||
|
curl -v \
|
||||||
|
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
||||||
|
http://192.168.88.106:3902/users-1002.json
|
||||||
|
|
||||||
|
|
||||||
|
# test with a browser
|
||||||
|
https://s3-web.mydomain.com/my_bucket_name/users-1002.json
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# ===================================================================
|
||||||
|
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
||||||
|
# ===================================================================
|
||||||
|
|
||||||
|
struct SimpleCredentials
|
||||||
|
access_key_id::String
|
||||||
|
secret_key::String
|
||||||
|
token::String
|
||||||
|
end
|
||||||
|
|
||||||
|
struct GarageConfig <: AWS.AbstractAWSConfig
|
||||||
|
endpoint::String
|
||||||
|
region::String
|
||||||
|
credentials::SimpleCredentials
|
||||||
|
end
|
||||||
|
|
||||||
|
# Required extensions for AWS.jl pipeline
|
||||||
|
AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
|
||||||
|
AWS.credentials(c::SimpleCredentials) = c
|
||||||
|
AWS.check_credentials(c::SimpleCredentials) = c
|
||||||
|
AWS.region(aws::GarageConfig) = aws.region
|
||||||
|
AWS.credentials(aws::GarageConfig) = aws.credentials
|
||||||
|
|
||||||
|
""" Generate the base URL for a Garage S3 service request.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `aws::GarageConfig`
|
||||||
|
The Garage configuration containing the endpoint.
|
||||||
|
- `service::String`
|
||||||
|
The AWS service name (e.g., `"s3"`).
|
||||||
|
- `region::String`
|
||||||
|
The AWS region string.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `String`: The stripped endpoint URL (trailing slashes removed).
|
||||||
|
"""
|
||||||
|
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
||||||
|
return strip(aws.endpoint, '/')
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Generate the full service URL including the resource path for a Garage S3 request.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `aws::GarageConfig`
|
||||||
|
The Garage configuration containing the endpoint.
|
||||||
|
- `service::String`
|
||||||
|
The AWS service name (e.g., `"s3"`).
|
||||||
|
- `resource::String`
|
||||||
|
The S3 resource path (e.g., `"/bucket/key"`).
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `String`: The full URL combining the endpoint and resource path.
|
||||||
|
"""
|
||||||
|
function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
|
||||||
|
endpoint = strip(aws.endpoint, '/')
|
||||||
|
resource_path = startswith(resource, '/') ? resource : "/" * resource
|
||||||
|
return string(endpoint, resource_path)
|
||||||
|
end
|
||||||
|
|
||||||
|
# ===================================================================
|
||||||
|
# 2. Thread-Safe Storage Client Wrapper
|
||||||
|
# ===================================================================
|
||||||
|
|
||||||
|
struct GarageStorage
|
||||||
|
config::GarageConfig
|
||||||
|
bucket::String
|
||||||
|
end
|
||||||
|
""" Create a thread-safe Garage S3 storage client.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `endpoint::String`
|
||||||
|
The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
|
||||||
|
- `key::String`
|
||||||
|
The Garage access key ID (created via garage-ui).
|
||||||
|
- `secret::String`
|
||||||
|
The secret key corresponding to the access key.
|
||||||
|
- `bucket::String`
|
||||||
|
The name of the S3 bucket to use.
|
||||||
|
|
||||||
|
# Keyword Arguments
|
||||||
|
- `region::String = "garage"`
|
||||||
|
The region identifier for the Garage instance.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `GarageStorage`: A thread-safe storage client instance.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "sommpanion-s3")
|
||||||
|
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
||||||
|
creds = SimpleCredentials(key, secret, "")
|
||||||
|
config = GarageConfig(endpoint, region, creds)
|
||||||
|
return GarageStorage(config, bucket)
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Upload an object to the Garage S3 bucket.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`
|
||||||
|
The storage client instance.
|
||||||
|
- `key::String`
|
||||||
|
The object key (name) in the bucket. Must not contain slashes (`/`).
|
||||||
|
- `data::Union{String, Vector{UInt8}}`
|
||||||
|
The data to upload.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `Nothing` if the key contains slashes (upload aborted with warning).
|
||||||
|
- Prints a success message if the upload completes.
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
||||||
|
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```julia
|
||||||
|
julia> put_file(storage, "test.json", "{\"status\": \"active\"}")
|
||||||
|
Successfully uploaded: test.json
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})
|
||||||
|
# Check if the key contains a slash (virtual folder character)
|
||||||
|
if occursin('/', key)
|
||||||
|
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
||||||
|
return nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
# Proceed if the key is flat
|
||||||
|
s3_put(storage.config, storage.bucket, key, data)
|
||||||
|
println("Successfully uploaded: ", key)
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Download an object from the Garage S3 bucket.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`
|
||||||
|
The storage client instance.
|
||||||
|
- `key::String`
|
||||||
|
The object key (name) in the bucket.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `Vector{UInt8}`: The raw bytes of the downloaded object.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```julia
|
||||||
|
julia> data = String(get_file(storage, "test.json"))
|
||||||
|
"\"{\\\"status\\\": \\\"active\\\"}\""
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
|
||||||
|
return s3_get(storage.config, storage.bucket, key)
|
||||||
|
end
|
||||||
|
|
||||||
|
""" List all object keys in the Garage S3 bucket.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`
|
||||||
|
The storage client instance.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `Vector{String}`: A list of object keys (file names) in the bucket.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```julia
|
||||||
|
julia> keys = list_files(storage)
|
||||||
|
3-element Vector{String}:
|
||||||
|
"test1.json"
|
||||||
|
"test2.json"
|
||||||
|
"users-1002.json"
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function list_files(storage::GarageStorage)
|
||||||
|
# Approach 2: s3_list_objects returns a Vector of Dicts with object details
|
||||||
|
objects = s3_list_objects(storage.config, storage.bucket)
|
||||||
|
return [obj["Key"] for obj in objects]
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Delete an object from the Garage S3 bucket.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`
|
||||||
|
The storage client instance.
|
||||||
|
- `key::String`
|
||||||
|
The object key (name) to delete.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```julia
|
||||||
|
julia> delete_file(storage, "test.json")
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function delete_file(storage::GarageStorage, key::String)
|
||||||
|
s3_delete(storage.config, storage.bucket, key)
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
end # module GarageS3
|
||||||
+463
-206
@@ -5,7 +5,7 @@ export noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue,
|
|||||||
replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul3Dto3DmanyTo1batch,
|
replaceLessThan, replaceBetween, cartesianAssign!, sumAlongDim3, matMul3Dto3DmanyTo1batch,
|
||||||
matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax,
|
matMul_3Dto4D_batchwise, isNotEqual, linearToCartesian, vectorMax, findMax,
|
||||||
multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween,
|
multiply_last, multiplyRandomElements, replaceElements, replaceElements!, isBetween,
|
||||||
isLess, allTrue, getStringBetweenCharacters, JSON3read_stringKey, mkDictPath!,
|
isLess, allTrue, getStringBetweenCharacters, mkDictPath!, dict_to_string_html,
|
||||||
getDictPath, detectKeywordVariation, textToDict, dictify, ordereddictify
|
getDictPath, detectKeywordVariation, textToDict, dictify, ordereddictify
|
||||||
|
|
||||||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV
|
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, CSV
|
||||||
@@ -150,14 +150,31 @@ end
|
|||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
""" read_textfile_by_index(folder_path::String, read_file_number::Integer=1)
|
""" Reads the x-th text file from a folder, where files are listed by the OS
|
||||||
|
without explicit sorting. Returns the file number, filename, and content.
|
||||||
|
|
||||||
with multiple text file in a folder,
|
# Arguments
|
||||||
this function read x_th text file in a folder (filename is sorted by OS)
|
- `folder_path::String`
|
||||||
|
Path to the folder containing text files.
|
||||||
|
- `read_file_number::Integer=1`
|
||||||
|
Which file to read (1-based index). Defaults to the first file.
|
||||||
|
|
||||||
# Example
|
# Return
|
||||||
utils.read_textfile_by_index(cleaned_data_path, 2)
|
- A tuple of `(file_number::Integer, filename::String, content)` where:
|
||||||
read 2nd txt file in a folder
|
- `file_number` is the index of the file that was read
|
||||||
|
- `filename` is the actual filename string
|
||||||
|
- `content` is a vector of lines from the file (or `nothing` if error)
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- Errors if `folder_path` is not a valid directory.
|
||||||
|
- Errors if `read_file_number` exceeds the number of files in the folder.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> result = read_textfile_by_index("/path/to/folder", 2)
|
||||||
|
(2, "sample.txt", ["line1", "line2", ...])
|
||||||
|
```
|
||||||
"""
|
"""
|
||||||
function read_textfile_by_index(folder_path::String, read_file_number::Integer=1)
|
function read_textfile_by_index(folder_path::String, read_file_number::Integer=1)
|
||||||
if isdir(folder_path)
|
if isdir(folder_path)
|
||||||
@@ -186,40 +203,39 @@ function read_textfile_by_index(folder_path::String, read_file_number::Integer=1
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary.
|
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into an OrderedDict.
|
||||||
|
|
||||||
# What it does
|
The function walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`,
|
||||||
- Walks any nested structure composed of `AbstractDict` (e.g., `JSON.Object`,
|
`Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where
|
||||||
`Dict`, `OrderedDict`) and `AbstractArray` and produces a new tree where
|
every dictionary-like node is an `OrderedDict` and every array-like node is a `Vector{Any}`.
|
||||||
every dictionary-like node is a plain `Dict` and every array-like node is a
|
Scalar values (numbers, strings, booleans, `nothing`, etc.) are returned unchanged.
|
||||||
`Vector{Any}`. Scalar values (numbers, strings, booleans, `nothing`, etc.)
|
Does **not** mutate the input; it always allocates new containers.
|
||||||
are returned unchanged.
|
|
||||||
- Does **not** mutate the input; it always allocates new containers.
|
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `x`
|
- `x`
|
||||||
Any Julia value. If `x` is an `AbstractDict` it will be converted to a `Dict`;
|
Any Julia value. If `x` is an `AbstractDict` it will be converted to an `OrderedDict`;
|
||||||
if it is an `AbstractArray` its elements will be processed recursively.
|
if it is an `AbstractArray` its elements will be processed recursively.
|
||||||
|
|
||||||
# Keyword Arguments
|
# Keyword Arguments
|
||||||
- `stringkey::Bool=false`
|
- `keytype::Type=Any`
|
||||||
If `true`, every dictionary key is converted to `String` via `string(k)`.
|
The key type for the output OrderedDict. Use `String` for `OrderedDict{String,Any}`,
|
||||||
If `false`, original key objects are preserved (useful when keys are already
|
`Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types.
|
||||||
`String`, `Symbol`, or other types you want to keep).
|
- `sort_order::Union{Nothing, Vector}=nothing`
|
||||||
|
Vector of keys specifying the desired order. Keys are arranged in the specified order
|
||||||
|
first, followed by any remaining keys.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- A newly allocated nested structure composed of `Dict{Any,Any}` and
|
- A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and `Vector{Any}`
|
||||||
`Vector{Any}` that mirrors the input shape but uses plain Julia containers.
|
that mirrors the input shape but uses ordered Julia containers.
|
||||||
|
|
||||||
# Notes
|
# Notes
|
||||||
- The function treats any `AbstractDict` as a mapping source, so it works with
|
- The function treats any `AbstractDict` as a mapping source, so it works with
|
||||||
`JSON.Object`, `Dict`, `OrderedDict`, etc.
|
`JSON.Object`, `Dict`, `OrderedDict`, etc.
|
||||||
- Arrays are returned as `Vector{Any}` with their elements processed
|
- Arrays are returned as `Vector{Any}` with their elements processed recursively.
|
||||||
recursively.
|
|
||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using JSON
|
julia> using JSON, DataStructures
|
||||||
julia> d = Dict(
|
julia> d = Dict(
|
||||||
"a" => 4,
|
"a" => 4,
|
||||||
"b" => 6,
|
"b" => 6,
|
||||||
@@ -234,92 +250,105 @@ julia> d = Dict(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
julia jsonstring = JSON.json(d)
|
julia> jsonstring = JSON.json(d)
|
||||||
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
||||||
julia> A2 = dictify(A1)
|
julia> A2 = dictify(A1; keytype=String)
|
||||||
Dict{Any,Any} with 3 entries:
|
OrderedDict{String,Any} with 3 entries:
|
||||||
"a" => 4
|
"a" => 4
|
||||||
"b" => 6
|
"b" => 6
|
||||||
"c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
"c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||||
|
|
||||||
julia> A3 = dictify(A1; stringkey=false) # preserves original key objects
|
julia> A3 = dictify(A1; keytype=Symbol)
|
||||||
julia> B1 = dictify(d; stringkey=true) # convert all keys in to string
|
OrderedDict{Symbol,Any} with 3 entries:
|
||||||
Dict{Any, Any} with 3 entries:
|
:a => 4
|
||||||
"c" => Dict{Any, Any}("e"=>Dict{Any, Any}("f"=>"hey", "g"=>Dict{Any, Any}("world"=>Any[1, "2", 3, Dict{Any, Any}("dd"=>4.7)])), "d"=>7)
|
:b => 6
|
||||||
"b" => 6
|
:c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||||
"a" => 4
|
|
||||||
|
julia> B1 = dictify(d; keytype=String)
|
||||||
|
OrderedDict{String, Any} with 3 entries:
|
||||||
|
```
|
||||||
|
|
||||||
|
**With sort_order:**
|
||||||
|
```jldoctest
|
||||||
|
julia> d = Dict("a"=>1, "b"=>2, "c"=>3)
|
||||||
|
julia> dictify(d; sort_order=["c", "a"])
|
||||||
|
OrderedDict{String,Int} with 3 entries:
|
||||||
|
"c" => 3
|
||||||
|
"a" => 1
|
||||||
|
"b" => 2
|
||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
function dictify(x; stringkey::Bool=false)
|
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||||
# Dict-like objects
|
)::OrderedDict where {T<:AbstractDict}
|
||||||
if x isa AbstractDict
|
# Dict-like objects
|
||||||
# choose output key type container (String keys when requested)
|
out = OrderedDict{keytype, Any}()
|
||||||
out = Dict{Any,Any}()
|
|
||||||
for (k,v) in x
|
# 1. Process and normalize all keys from the input dictionary
|
||||||
newk = stringkey ? string(k) : k
|
processed_dict = OrderedDict{keytype, Any}()
|
||||||
out[newk] = dictify(v; stringkey=stringkey)
|
for (k, v) in x
|
||||||
end
|
if keytype === String
|
||||||
return out
|
newk = string(k)
|
||||||
# Arrays / vectors: map elements recursively and return a Vector{Any}
|
elseif keytype === Symbol
|
||||||
elseif x isa AbstractArray
|
newk = Symbol(string(k))
|
||||||
return [dictify(element; stringkey=stringkey) for element in x]
|
else
|
||||||
# everything else: return as-is (primitives, numbers, strings, etc.)
|
newk = k
|
||||||
else
|
end
|
||||||
return x
|
processed_dict[newk] = dictify(v; keytype=keytype, sort_order=sort_order)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# 2. If a sort order is specified, apply it
|
||||||
|
if !isnothing(sort_order)
|
||||||
|
# Normalize the sort_order elements to match the requested keytype
|
||||||
|
normalized_order = map(sort_order) do tk
|
||||||
|
if keytype === String
|
||||||
|
return string(tk)
|
||||||
|
elseif keytype === Symbol
|
||||||
|
return Symbol(string(tk))
|
||||||
|
else
|
||||||
|
return tk
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# First, insert keys that match the requested order
|
||||||
|
for target_key in normalized_order
|
||||||
|
if haskey(processed_dict, target_key)
|
||||||
|
out[target_key] = processed_dict[target_key]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Then, append any remaining keys that weren't in the sort_order
|
||||||
|
for (k, v) in processed_dict
|
||||||
|
if !haskey(out, k)
|
||||||
|
out[k] = v
|
||||||
|
end
|
||||||
|
end
|
||||||
|
else
|
||||||
|
# If no sort order is given, just use the processed dict
|
||||||
|
out = processed_dict
|
||||||
|
end
|
||||||
|
|
||||||
|
return out
|
||||||
end
|
end
|
||||||
|
|
||||||
|
function dictify(x::T; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||||
|
) where {T<:AbstractArray}
|
||||||
|
return [dictify(element; keytype=keytype, sort_order=sort_order) for element in x]
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
|
||||||
|
|
||||||
""" Array_to_JSON_str(data::AbstractArray)
|
|
||||||
|
|
||||||
encode Array to JSON String
|
|
||||||
|
|
||||||
# Example
|
|
||||||
|
|
||||||
a = [1.23 4.7889; 9987.1 -123.07; -0.0027 -6.75]
|
|
||||||
jsonStr = Array_to_JSON_str(a)
|
|
||||||
jsonStr = "{\"Array\":[[1.23,9987.1,-0.0027],[4.7889,-123.07,-6.75]],\"size\":[3,2]}"
|
|
||||||
"""
|
|
||||||
function Array_to_JSON_str(data::AbstractArray)
|
|
||||||
d = Dict("Array"=> data, "size"=>size(data))
|
|
||||||
jsonStr = JSON.json(d)
|
|
||||||
return jsonStr
|
|
||||||
end
|
end
|
||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
function dictify(x; keytype::Type=Any, sort_order::Union{Nothing, Vector}=nothing
|
||||||
|
)
|
||||||
""" JSON_str_to_Array(json_str::String)
|
return x
|
||||||
|
|
||||||
decode JSON String to Array
|
|
||||||
|
|
||||||
# Example
|
|
||||||
|
|
||||||
jsonStr = "{\"Array\":[[1.23,9987.1,-0.0027],[4.7889,-123.07,-6.75]],\"size\":[3,2]}"
|
|
||||||
a = JSON_str_to_Array(jsonStr)
|
|
||||||
|
|
||||||
"""
|
|
||||||
function JSON_str_to_Array(jsonStr::String)
|
|
||||||
jsonObj = JSON.parse(jsonStr)
|
|
||||||
a = Array(jsonObj.Array)
|
|
||||||
array = hcat(a...)
|
|
||||||
return array
|
|
||||||
end
|
end
|
||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary.
|
""" Recursively convert dictionary-like variable (e.g. JSON.Object) into a dictionary.
|
||||||
|
The function walks any nested structure composed of AbstractDict (e.g., JSON.Object,
|
||||||
# What it does
|
Dict, OrderedDict) and AbstractArray and produces a new tree where
|
||||||
- Walks any nested structure composed of AbstractDict (e.g., JSON.Object,
|
every dictionary-like node is an OrderedDict{Any,Any} and every array-like
|
||||||
Dict, OrderedDict) and AbstractArray and produces a new tree where
|
node is a Vector{Any}. Scalar values (numbers, strings, booleans,
|
||||||
every dictionary-like node is an OrderedDict{Any,Any} and every array-like
|
nothing, etc.) are returned unchanged.
|
||||||
node is a Vector{Any}. Scalar values (numbers, strings, booleans,
|
Does **not** mutate the input; it always allocates new containers.
|
||||||
nothing, etc.) are returned unchanged.
|
|
||||||
- Does **not** mutate the input; it always allocates new containers.
|
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `x`
|
- `x`
|
||||||
@@ -328,13 +357,11 @@ end
|
|||||||
processed recursively.
|
processed recursively.
|
||||||
|
|
||||||
# Keyword Arguments
|
# Keyword Arguments
|
||||||
- `stringkey::Bool=false`
|
- `keytype::Type=Any`
|
||||||
If `true`, every dictionary key is converted to `String` via `string(k)`.
|
The key type for the output Dict. Use `String` for `OrderedDict{String,Any}`, `Symbol` for `OrderedDict{Symbol,Any}`, or `Any` to preserve original key types.
|
||||||
If `false`, original key objects are preserved (useful when keys are already
|
|
||||||
`String`, `Symbol`, or other types you want to keep).
|
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- A newly allocated nested structure composed of `OrderedDict{Any,Any}` and
|
- A newly allocated nested structure composed of `OrderedDict{keytype,Any}` and
|
||||||
`Vector{Any}` that mirrors the input shape but uses ordered Julia containers.
|
`Vector{Any}` that mirrors the input shape but uses ordered Julia containers.
|
||||||
|
|
||||||
# Notes
|
# Notes
|
||||||
@@ -361,41 +388,52 @@ julia> d = Dict(
|
|||||||
|
|
||||||
julia jsonstring = JSON.json(d)
|
julia jsonstring = JSON.json(d)
|
||||||
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
julia> A1 = JSON.parse(jsonstring) # A1 type is JSON.Object
|
||||||
julia> A2 = OrderedDict(A1)
|
julia> A2 = ordereddictify(A1; keytype=String)
|
||||||
Dict{Any,Any} with 3 entries:
|
OrderedDict{String,Any} with 3 entries:
|
||||||
"a" => 4
|
"a" => 4
|
||||||
"b" => 6
|
"b" => 6
|
||||||
"c" => Dict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
"c" => OrderedDict("d"=>7, "e"=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||||
|
|
||||||
julia> A3 = OrderedDict(A1; stringkey=false) # preserves original key objects
|
julia> A3 = ordereddictify(A1; keytype=Symbol)
|
||||||
julia> B1 = OrderedDict(d; stringkey=true) # convert all keys in to string
|
OrderedDict{Symbol,Any} with 3 entries:
|
||||||
Dict{Any, Any} with 3 entries:
|
:a => 4
|
||||||
"c" => Dict{Any, Any}("e"=>Dict{Any, Any}("f"=>"hey", "g"=>Dict{Any, Any}("world"=>Any[1, "2", 3, Dict{Any, Any}("dd"=>4.7)])), "d"=>7)
|
:b => 6
|
||||||
|
:c => OrderedDict(:d=>7, :e=>Dict("f"=>"hey", "g"=>Dict("world"=>[1, "2", 3, 4.7])))
|
||||||
|
|
||||||
|
julia> B1 = ordereddictify(d; keytype=String)
|
||||||
|
OrderedDict{String, Any} with 3 entries:
|
||||||
|
"c" => OrderedDict{String, Any}("e"=>OrderedDict{String, Any}("f"=>"hey", "g"=>OrderedDict{String, Any}("world"=>Any[1, "2", 3, OrderedDict{String, Any}("dd"=>4.7)])), "d"=>7)
|
||||||
"b" => 6
|
"b" => 6
|
||||||
"a" => 4
|
"a" => 4
|
||||||
```
|
```
|
||||||
Ref. https://github.com/andyferris/Dictionaries.jl
|
Ref. https://github.com/andyferris/Dictionaries.jl
|
||||||
"""
|
"""
|
||||||
function ordereddictify(x; stringkey::Bool=false)
|
function ordereddictify(x; keytype::Type=Any)
|
||||||
# Dict-like objects
|
# Dict-like objects
|
||||||
if x isa AbstractDict
|
if x isa AbstractDict
|
||||||
# choose output key type container (String keys when requested)
|
# choose output key type container
|
||||||
out = OrderedDict{Any,Any}()
|
out = OrderedDict{keytype,Any}()
|
||||||
for (k,v) in x
|
for (k,v) in x
|
||||||
newk = stringkey ? string(k) : k
|
if keytype === String
|
||||||
out[newk] = ordereddictify(v; stringkey=stringkey)
|
newk = string(k)
|
||||||
|
elseif keytype === Symbol
|
||||||
|
newk = Symbol(string(k))
|
||||||
|
else
|
||||||
|
newk = k
|
||||||
|
end
|
||||||
|
out[newk] = ordereddictify(v; keytype=keytype)
|
||||||
end
|
end
|
||||||
return out
|
return out
|
||||||
# Arrays / vectors: map elements recursively and return a Vector{Any}
|
# Arrays / vectors: map elements recursively and return a Vector{Any}
|
||||||
elseif x isa AbstractArray
|
elseif x isa AbstractArray
|
||||||
return [ordereddictify(element; stringkey=stringkey) for element in x]
|
return [ordereddictify(element; keytype=keytype) for element in x]
|
||||||
# everything else: return as-is (primitives, numbers, strings, etc.)
|
# everything else: return as-is (primitives, numbers, strings, etc.)
|
||||||
else
|
else
|
||||||
return x
|
return x
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
#------------------------------------------------------------------------------------------------100
|
#----------------------------------------------100---------------------------------------------
|
||||||
|
|
||||||
"""
|
"""
|
||||||
print time of cpu executtion at the line inwhich this macro is used
|
print time of cpu executtion at the line inwhich this macro is used
|
||||||
@@ -477,6 +515,32 @@ function randomChoiceOnTarget(target::AbstractVector, choiceList::AbstractVector
|
|||||||
return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,))
|
return randomChoiceOnTarget.(target, 1, (choiceList,), (probability,))
|
||||||
end
|
end
|
||||||
|
|
||||||
|
""" Compute the linearly weighted average of an array.
|
||||||
|
The function assigns weights proportional to position indices (1, 2, 3, ...) to array
|
||||||
|
elements and returns the weighted average.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `a::Array`
|
||||||
|
Array of numeric values. Elements must support multiplication with numbers
|
||||||
|
and summation.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- The linearly weighted average as a floating-point number.
|
||||||
|
|
||||||
|
# Formula
|
||||||
|
For an array `a` with `n` elements, computes:
|
||||||
|
```
|
||||||
|
sum(i * a[i]) / sum(a) for i = 1 to n
|
||||||
|
```
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> a = [10, 20, 30]
|
||||||
|
julia> linearly_weighted_avg(a)
|
||||||
|
23.333333333333332
|
||||||
|
```
|
||||||
|
"""
|
||||||
function linearly_weighted_avg(a::Array)
|
function linearly_weighted_avg(a::Array)
|
||||||
total = 0.0
|
total = 0.0
|
||||||
for (i, v) in enumerate(a)
|
for (i, v) in enumerate(a)
|
||||||
@@ -487,10 +551,34 @@ function linearly_weighted_avg(a::Array)
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
""" Convert String that is holded inside a variable to Symbol
|
""" Convert a variable's value (String) into a Symbol.
|
||||||
# Example
|
The function takes a variable containing a String value and converts it to a Symbol
|
||||||
x = "hello" # x is a variable holding String "hello" \n
|
using Julia's expression interpolation mechanism.
|
||||||
y = variable_to_symbol(x) # y holds :hello
|
|
||||||
|
# Arguments
|
||||||
|
- `variable`
|
||||||
|
Any variable whose value is a String. The function uses `string(variable)`
|
||||||
|
internally to obtain the value.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- A `Symbol` constructed from the string value of the input variable.
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- This function uses variable interpolation to capture the variable's
|
||||||
|
value as a Symbol. It works with any variable type that can be converted
|
||||||
|
to String.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> x = "hello"
|
||||||
|
julia> variable_str_to_symbol(x)
|
||||||
|
:hello
|
||||||
|
|
||||||
|
julia> y = "world_test"
|
||||||
|
julia> variable_str_to_symbol(y)
|
||||||
|
:world_test
|
||||||
|
```
|
||||||
"""
|
"""
|
||||||
function variable_str_to_symbol(variable)
|
function variable_str_to_symbol(variable)
|
||||||
semi = :($variable)
|
semi = :($variable)
|
||||||
@@ -534,6 +622,43 @@ function fieldname_useable_type(somestruct, fieldname::Symbol;
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" Draw unique elements from a list without replacement.
|
||||||
|
The function randomly selects a specified number of distinct elements from a collection,
|
||||||
|
optionally excluding certain elements from consideration. Uses in-place
|
||||||
|
shuffling for efficiency.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `drawOptions::Array`
|
||||||
|
Collection of elements to draw from.
|
||||||
|
- `draw_number::Integer`
|
||||||
|
Number of unique elements to draw.
|
||||||
|
|
||||||
|
# Keyword Arguments
|
||||||
|
- `exclude_list::Union{AbstractArray,Nothing}=nothing`
|
||||||
|
Elements to exclude from the drawing pool. If `nothing`, no elements are
|
||||||
|
excluded.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- An array of `draw_number` unique elements drawn from `drawOptions`, excluding
|
||||||
|
any elements in `exclude_list`.
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- The function copies `drawOptions` and shuffles in-place, then pops elements
|
||||||
|
sequentially to ensure uniqueness.
|
||||||
|
- Errors if `draw_number` exceeds the number of available elements after
|
||||||
|
exclusion.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> options = [1, 2, 3, 4, 5]
|
||||||
|
julia> randomNoRepeat(options, 3)
|
||||||
|
[3, 1, 5]
|
||||||
|
|
||||||
|
julia> randomNoRepeat(options, 2; exclude_list=[1, 5])
|
||||||
|
[4, 2]
|
||||||
|
```
|
||||||
|
"""
|
||||||
function randomNoRepeat(drawOptions::Array, draw_number::Integer;
|
function randomNoRepeat(drawOptions::Array, draw_number::Integer;
|
||||||
exclude_list::Union{AbstractArray,Nothing}=nothing)
|
exclude_list::Union{AbstractArray,Nothing}=nothing)
|
||||||
draw_option = copy(drawOptions)
|
draw_option = copy(drawOptions)
|
||||||
@@ -700,22 +825,46 @@ function selectRange(d::Dict{Symbol, <:AbstractVector}, range)
|
|||||||
return newDict
|
return newDict
|
||||||
end
|
end
|
||||||
|
|
||||||
""" Assign value to a given Dict by array of keys
|
""" Recursively traverses a nested dictionary structure using a vector of keys
|
||||||
|
and assigns a value to the final key. Creates intermediate dictionaries
|
||||||
|
if they don't exist.
|
||||||
|
|
||||||
# Example
|
# Arguments
|
||||||
d = Dict(
|
- `dict::Dict`
|
||||||
:a1=> Dict(:c=> 5),
|
The root dictionary to traverse and modify.
|
||||||
:a2=> Dict(
|
- `accessArray::Array{Symbol}`
|
||||||
:k=> 10,
|
A vector of symbols representing the key path to traverse.
|
||||||
:b=> Dict(
|
- `valueToAssign`
|
||||||
:s=> "target",
|
The value to assign at the final key in the path.
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
index = [:a2, :b, :s] \n
|
|
||||||
assignDict!(d, [:a2, :b, :s], "wow")
|
|
||||||
|
|
||||||
return 1 if no target key in a given dict.
|
# Return
|
||||||
|
- `0` on success (value assigned)
|
||||||
|
- `1` if the path cannot be traversed (missing intermediate keys)
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- The function walks through each key in `accessArray` except the last one,
|
||||||
|
expecting intermediate keys to already exist in the dictionary.
|
||||||
|
- If any intermediate key is missing, the function returns `1` without
|
||||||
|
modifying the dictionary.
|
||||||
|
- The final key in `accessArray` receives the `valueToAssign`.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> d = Dict(
|
||||||
|
:a1=> Dict(:c=> 5),
|
||||||
|
:a2=> Dict(
|
||||||
|
:k=> 10,
|
||||||
|
:b=> Dict(
|
||||||
|
:s=> "target",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
julia> assignDict!(d, [:a2, :b, :s], "wow")
|
||||||
|
0
|
||||||
|
julia> d[:a2][:b][:s]
|
||||||
|
"wow"
|
||||||
|
```
|
||||||
"""
|
"""
|
||||||
function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign)
|
function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign)
|
||||||
wd = nothing
|
wd = nothing
|
||||||
@@ -736,9 +885,39 @@ function assignDict!(dict::Dict, accessArray::Array{Symbol}, valueToAssign)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
""" convert hour(0-23), minute(0-59) into julia time object
|
""" Converts hour (0-23) and minute (0-59) into a Julia `Time` object using
|
||||||
# Example
|
12-hour format with AM/PM indicator.
|
||||||
time
|
|
||||||
|
# Arguments
|
||||||
|
- `h::Integer`
|
||||||
|
Hour in 24-hour format (0 to 23).
|
||||||
|
- `m::Integer`
|
||||||
|
Minute (0 to 59).
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- A `Time` object representing the time in 12-hour format with AM/PM.
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- Hours 0 and 12 are special cases: 0 becomes 12 AM, 12 becomes 12 PM.
|
||||||
|
- Hours 1-11 remain the same with "am" suffix.
|
||||||
|
- Hours 13-23 are converted to 1-11 with "pm" suffix.
|
||||||
|
- Minutes less than 10 are zero-padded.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> iTime(0, 30)
|
||||||
|
12:30 AM
|
||||||
|
|
||||||
|
julia> iTime(9, 15)
|
||||||
|
9:15 AM
|
||||||
|
|
||||||
|
julia> iTime(12, 0)
|
||||||
|
12:00 PM
|
||||||
|
|
||||||
|
julia> iTime(14, 5)
|
||||||
|
2:05 PM
|
||||||
|
```
|
||||||
"""
|
"""
|
||||||
function iTime(h::Integer, m::Integer)
|
function iTime(h::Integer, m::Integer)
|
||||||
if h == 0
|
if h == 0
|
||||||
@@ -782,19 +961,37 @@ function limitvalue(v::Number, lowerbound::Pair, upperbound::Pair)
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
""" Assign matrix b to matrix a according to matrix b's CartesianIndex.
|
""" Assigns elements from matrix `b` to matrix `a` using the Cartesian indices
|
||||||
|
of `b`. Elements are copied in the order they appear when iterating over `b`,
|
||||||
|
and placed into `a` at the corresponding Cartesian positions of `b`.
|
||||||
|
|
||||||
Arguments:\n
|
# Arguments
|
||||||
a : target matrix.
|
- `a`
|
||||||
b : source matrix.
|
Target matrix where values from `b` will be assigned.
|
||||||
|
- `b`
|
||||||
|
Source matrix whose Cartesian indices determine where values are placed in `a`.
|
||||||
|
|
||||||
Return:\n
|
# Return
|
||||||
Resulting matrix a.
|
- `nothing`
|
||||||
|
|
||||||
Example:\n
|
# Notes
|
||||||
```jldoctest
|
- The function iterates through `b` in column-major order (Julia's default),
|
||||||
julia> not done yet
|
retrieving each element's Cartesian index and assigning it to the same
|
||||||
```
|
position in `a`.
|
||||||
|
- Matrix `a` must have sufficient size to accommodate all Cartesian indices
|
||||||
|
from `b`; otherwise, an `BoundsError` may occur.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> a = zeros(4, 4);
|
||||||
|
julia> b = [1 2; 3 4];
|
||||||
|
julia> cartesianAssign!(a, b);
|
||||||
|
julia> a[1:2, 1:2]
|
||||||
|
2×2 Matrix{Float64}:
|
||||||
|
1.0 3.0
|
||||||
|
2.0 4.0
|
||||||
|
```
|
||||||
"""
|
"""
|
||||||
function cartesianAssign!(a, b)
|
function cartesianAssign!(a, b)
|
||||||
for (i, v) in enumerate(b)
|
for (i, v) in enumerate(b)
|
||||||
@@ -1149,7 +1346,6 @@ julia> text = "{\"ask\": {\"text\": \"Could you please tell me about the special
|
|||||||
julia> GeneralUtils.getStringBetweenCharacters(text, '{', '}', endCharLocation="end")
|
julia> GeneralUtils.getStringBetweenCharacters(text, '{', '}', endCharLocation="end")
|
||||||
"{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}"
|
"{\"ask\": {\"text\": \"Could you please tell me about the special event?\"\n}}"
|
||||||
```
|
```
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char;
|
function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char;
|
||||||
endCharLocation::String="next", includeChar::Bool=true)::String where {T<:AbstractString}
|
endCharLocation::String="next", includeChar::Bool=true)::String where {T<:AbstractString}
|
||||||
@@ -1181,59 +1377,40 @@ function getStringBetweenCharacters(text::T, startChar::Char, endChar::Char;
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
|
""" Recursively creates nested dictionary paths if they do not exist and assigns
|
||||||
""" Read JSON string and return a dictionary with string key. (JSON3 defaults to symbol key)
|
a value to the final key. Similar to `mkpath()` but for dictionaries.
|
||||||
This function solve the problem of reading JSON with string key.
|
|
||||||
|
|
||||||
Arguments:
|
|
||||||
jsonString::String
|
|
||||||
|
|
||||||
Return:
|
|
||||||
a dictionary with string key
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```jldoctest
|
|
||||||
julia> jsonString = {\"wine type\": \"Red\", \"intensity level\": \"medium-bodied\"}
|
|
||||||
julia> JSON3read_stringKey(jsonString)
|
|
||||||
Dict{String, Any} with 2 entries:
|
|
||||||
"intensity level" => "medium-bodied"
|
|
||||||
"wine type" => "Red"
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
function JSON3read_stringKey(jsonString::AbstractString)
|
|
||||||
jsonobj = JSON3.read(jsonString)
|
|
||||||
newDict = OrderedDict{String,Any}()
|
|
||||||
for (k,v) in jsonobj
|
|
||||||
newDict[string(k)] = v
|
|
||||||
end
|
|
||||||
return newDict
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
""" Create nested dict path if it does not already exist. The same concept as Julia's mkpath()
|
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `dict::Dict`
|
- `dict::Union{Dict{Symbol, Any}, Dict{String, Any}}`
|
||||||
target dict
|
The target dictionary to traverse and modify. Must use consistent key types
|
||||||
- `addkeys::Union{Vector{String}, Vector{Symbol}}`
|
(either all `String` or all `Symbol`).
|
||||||
keys to be added to dict
|
- `addkeys::Union{Vector{String}, Vector{Symbol}}`
|
||||||
- `value`
|
A vector of keys representing the path to traverse. Intermediate dictionaries
|
||||||
value to be added to dict at final key in keypath
|
are created if they don't exist.
|
||||||
|
- `value`
|
||||||
|
The value to assign at the final key in the path.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- dict with added keypath
|
- The assigned `value`.
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
|
- The function ensures key type consistency: the type of keys being added must
|
||||||
|
match the type of existing keys in the dictionary.
|
||||||
|
- Intermediate dictionaries are automatically created with the appropriate key
|
||||||
|
type when they don't exist.
|
||||||
|
- The function walks through each key in `addkeys` except the last one,
|
||||||
|
creating intermediate dictionaries as needed, and assigns `value` to the final
|
||||||
|
key.
|
||||||
|
|
||||||
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using Revise
|
|
||||||
julia> using GeneralUtils
|
julia> using GeneralUtils
|
||||||
julia> d = Dict{String, Any}("a" => Dict{String, Any}("b" => 10))
|
julia> d = Dict("a" => Dict("b" => 10))
|
||||||
julia> GeneralUtils.mkDictPath!(d, ["a", "v", "x", "y", "z"], 42)
|
julia> mkDictPath!(d, ["a", "v", "x", "y", "z"], 42)
|
||||||
Dict{String, Any} with 1 entry:
|
42
|
||||||
"path" => Dict{Any, Any}("to"=>Dict{Any, Any}("nested"=>Dict{Any, Any}("value"=>42)))
|
julia> d["a"]["v"]["x"]["y"]["z"]
|
||||||
|
42
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}},
|
function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}},
|
||||||
addkeys::Union{Vector{String}, Vector{Symbol}}, value)
|
addkeys::Union{Vector{String}, Vector{Symbol}}, value)
|
||||||
@@ -1257,27 +1434,33 @@ function mkDictPath!(dict::Union{Dict{Symbol, Any}, Dict{String, Any}},
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
""" Get nested dict value using a vector of keys
|
""" Retrieves a value from a nested dictionary by traversing a vector of keys.
|
||||||
|
Creates intermediate dictionaries if they don't exist.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `dict::Dict`
|
- `dict::Dict`
|
||||||
target dict
|
The root dictionary to traverse.
|
||||||
- `keys::Vector`
|
- `keys::Vector`
|
||||||
keys vector
|
A vector of keys representing the path to traverse. Each key in the vector
|
||||||
|
is used to access the next level of nesting.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- dict with added keypath
|
- The value at the final key in the path.
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
|
- Errors with `ArgumentError` if any intermediate key is missing from the
|
||||||
|
dictionary path.
|
||||||
|
- The function walks through each key in `keys` except the last one,
|
||||||
|
expecting intermediate keys to exist in the dictionary.
|
||||||
|
- The final key in `keys` is used to retrieve the value.
|
||||||
|
|
||||||
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using Revise
|
|
||||||
julia> using GeneralUtils
|
julia> using GeneralUtils
|
||||||
julia> d = Dict{Symbol, Any}(:a => Dict{Symbol, Any}(:b => 10))
|
julia> d = Dict(:a => Dict(:b => 10))
|
||||||
julia> GeneralUtils.getDictPath(d, [:a, :b])
|
julia> getDictPath(d, [:a, :b])
|
||||||
10
|
10
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function getDictPath(dict::Dict, keys::Vector)
|
function getDictPath(dict::Dict, keys::Vector)
|
||||||
current_dict = dict
|
current_dict = dict
|
||||||
@@ -1429,8 +1612,6 @@ OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do",
|
|||||||
:plan => "wake up and going out",
|
:plan => "wake up and going out",
|
||||||
:action => "1. wake up 2. eat 3. sleep")
|
:action => "1. wake up 2. eat 3. sleep")
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function textToDict(text::String, detectKeywords::Vector{String};
|
function textToDict(text::String, detectKeywords::Vector{String};
|
||||||
dictKey::Union{Vector{String}, Nothing}=nothing,
|
dictKey::Union{Vector{String}, Nothing}=nothing,
|
||||||
@@ -1491,6 +1672,82 @@ function textToDict(text::String, detectKeywords::Vector{String};
|
|||||||
return od2
|
return od2
|
||||||
end
|
end
|
||||||
|
|
||||||
|
""" Recursively convert dictionary into an HTML string representation.
|
||||||
|
The function walks a nested `AbstractDict` structure and produces a well-formed HTML string
|
||||||
|
where each dictionary key becomes an HTML tag. Nested dictionaries become
|
||||||
|
nested tags, and scalar values (numbers, strings, etc.) become the text
|
||||||
|
content of leaf tags.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `d::AbstractDict`
|
||||||
|
The dictionary to convert. Keys must be strings or symbols that form valid
|
||||||
|
HTML tag names.
|
||||||
|
|
||||||
|
# Keyword Arguments
|
||||||
|
- `indent_level::Integer=1`
|
||||||
|
Initial indentation level for the output. Each recursive level increases
|
||||||
|
indentation by one.
|
||||||
|
- `indent_str::String=" "`
|
||||||
|
String used for each indentation level.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- A single HTML string representing the dictionary structure, with proper
|
||||||
|
opening and closing tags and appropriate indentation.
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- Keys are sorted alphabetically for deterministic output.
|
||||||
|
- Works recursively: dictionary values produce nested tags; non-dictionary
|
||||||
|
values are placed as text between opening/closing tags.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> d = Dict(
|
||||||
|
"html" => Dict(
|
||||||
|
"head" => Dict("title" => "Test"),
|
||||||
|
"body" => Dict(
|
||||||
|
"h1" => "Hello",
|
||||||
|
"p" => "World"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
julia> println(dict_to_string_html(d))
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Test</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Hello</h1>
|
||||||
|
<p>World</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function dict_to_string_html(d::AbstractDict; indent_level=1, indent_str=" ")
|
||||||
|
lines = String[]
|
||||||
|
padding = indent_str ^ indent_level
|
||||||
|
|
||||||
|
for k in keys(d)
|
||||||
|
v = d[k]
|
||||||
|
|
||||||
|
if v isa AbstractDict
|
||||||
|
# Open tag, recurse for children, then close tag
|
||||||
|
push!(lines, "$padding<$k>")
|
||||||
|
ind_level = indent_level + 1
|
||||||
|
push!(lines, dict_to_string_html(v; indent_level=ind_level, indent_str=indent_str))
|
||||||
|
push!(lines, "$padding</$k>")
|
||||||
|
else
|
||||||
|
# Leaf node: put key and value on a single line
|
||||||
|
push!(lines, "$padding<$k>$v</$k>")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return join(lines, "\n")
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1002
-335
File diff suppressed because it is too large
Load Diff
+140
-85
@@ -5,7 +5,7 @@ export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, rep
|
|||||||
dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString,
|
dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString,
|
||||||
dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
|
dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
|
||||||
dictToString_numbering, extract_triple_backtick_text,
|
dictToString_numbering, extract_triple_backtick_text,
|
||||||
countGivenWords, remove_french_accents,
|
countGivenWords, remove_french_accents, removestring,
|
||||||
extractTextBetweenCharacter, extractTextBetweenString,
|
extractTextBetweenCharacter, extractTextBetweenString,
|
||||||
convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
|
convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
|
||||||
|
|
||||||
@@ -13,31 +13,34 @@ using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames
|
|||||||
|
|
||||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
""" Compute time different between start time and stop time in a given unit.
|
""" Computes the time difference between two `DateTime` values and returns the
|
||||||
Unit can be "milliseconds", "seconds", "minutes", "hours".
|
result in a specified unit: milliseconds, seconds, minutes, or hours.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `starttime::DateTime`
|
- `starttime::DateTime`
|
||||||
start time
|
The starting `DateTime` value.
|
||||||
- `stoptime::DateTime`
|
- `stoptime::DateTime`
|
||||||
stop time
|
The ending `DateTime` value.
|
||||||
- `unit::String`
|
- `unit::String`
|
||||||
unit of time difference
|
The unit for the result. Must be one of: `"milliseconds"`, `"seconds"`,
|
||||||
|
`"minutes"`, `"hours"`.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- time difference in given unit
|
- `Integer`: The time difference converted to the specified unit.
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
|
- The function computes `stoptime - starttime` and converts the result to the
|
||||||
|
requested unit using integer division.
|
||||||
|
- Errors with `ArgumentError` if an invalid unit is specified.
|
||||||
|
|
||||||
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using Revise
|
|
||||||
julia> using GeneralUtils, Dates
|
julia> using GeneralUtils, Dates
|
||||||
julia> a = Dates.now()
|
julia> a = Dates.now()
|
||||||
julia> b = a + Dates.Day(5) # add 5 days
|
julia> b = a + Dates.Day(5)
|
||||||
julia> GeneralUtils.timedifference(a, b, "hours")
|
julia> timedifference(a, b, "hours")
|
||||||
120
|
120
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer
|
function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer
|
||||||
diff = stoptime - starttime
|
diff = stoptime - starttime
|
||||||
@@ -184,21 +187,27 @@ end
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
""" Get uuid4 with snake case
|
""" Generates a UUID4 (version 4) identifier and converts it to snake case by
|
||||||
|
replacing hyphens with underscores.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- This function takes no arguments.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- `uuid4::String`
|
- `String`: A UUID4 string with underscores instead of hyphens (e.g.,
|
||||||
uuid4 with snake case
|
`"0f6e4f_568c_4df4_8c79_1d7a58072f4a"`).
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
|
- Uses the `uuid4()` function from the UUIDs standard library to generate a
|
||||||
|
random UUID.
|
||||||
|
- The underscore character replaces all hyphens in the UUID string.
|
||||||
|
|
||||||
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using Revise
|
|
||||||
julia> using GeneralUtils
|
julia> using GeneralUtils
|
||||||
julia> GeneralUtils.uuid4snakecase()
|
julia> uuid4snakecase()
|
||||||
"0f6e4f_568c_4df4_8c79_1d7a58072f4a"
|
"0f6e4f_568c_4df4_8c79_1d7a58072f4a"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function uuid4snakecase()::String
|
function uuid4snakecase()::String
|
||||||
_id = string(uuid4())
|
_id = string(uuid4())
|
||||||
@@ -207,32 +216,37 @@ function uuid4snakecase()::String
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
""" Replace a dictionary key with the new key
|
""" Replaces keys in a dictionary according to a mapping, returning a new
|
||||||
|
dictionary with updated keys while preserving the original values.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `d::Dict`
|
- `d::Dict`
|
||||||
The input dictionary that you want to modify
|
The input dictionary to modify.
|
||||||
- `replacementMap::Dict`
|
- `replacementMap::Dict`
|
||||||
A dictionary that maps old keys to new keys
|
A dictionary mapping old keys to new keys. Keys not present in this map are
|
||||||
|
left unchanged.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- `newDict::Dict`
|
- `Dict`: A new dictionary with replaced keys. Values are preserved from the
|
||||||
new dictionary with the replaced keys
|
original dictionary.
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
|
- The function creates a new dictionary rather than modifying the input in
|
||||||
|
place.
|
||||||
|
- Keys not found in `replacementMap` are copied to the result with their
|
||||||
|
original keys unchanged.
|
||||||
|
|
||||||
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using Revise
|
|
||||||
julia> using GeneralUtils
|
julia> using GeneralUtils
|
||||||
julia> d = Dict(:a => 1, :b => 2, :c => 3)
|
julia> d = Dict(:a => 1, :b => 2, :c => 3)
|
||||||
julia> replacement_map = Dict(:a => :x, :b => :y)
|
julia> replacement_map = Dict(:a => :x, :b => :y)
|
||||||
julia> new_dict = GeneralUtils.replaceDictKeys(d, replacement_map)
|
julia> replaceDictKeys(d, replacement_map)
|
||||||
Dict{Any, Any} with 3 entries:
|
Dict{Any, Any} with 3 entries:
|
||||||
:y => 2
|
:y => 2
|
||||||
:c => 3
|
:c => 3
|
||||||
:x => 1
|
:x => 1
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
|
function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
|
||||||
newDict = Dict()
|
newDict = Dict()
|
||||||
@@ -294,36 +308,43 @@ end
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
""" Execute a function with timer.
|
""" Executes a function with a timeout mechanism. If the function does not
|
||||||
|
complete within the specified time, it is interrupted and a timeout message
|
||||||
|
is returned.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `f::Function`
|
- `f::Function`
|
||||||
a function to run
|
The function to execute.
|
||||||
- `timeoutwindow::Integer``
|
- `timeoutwindow::Integer`
|
||||||
timeout in seconds
|
The timeout duration in seconds.
|
||||||
|
|
||||||
# Keyword Argument
|
# Keyword Arguments
|
||||||
- `fargs`
|
- `fargs`
|
||||||
arguments for the function
|
Arguments to pass to the function `f`. If `nothing`, the function is called
|
||||||
- `timeoutmsg::String`
|
without arguments.
|
||||||
time out message
|
- `timeoutmsg::String`
|
||||||
|
The message to return if the function times out. Defaults to `"task timed out"`.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- task result otherwise timeout message
|
- The result of the function if it completes within the timeout, otherwise the
|
||||||
|
`timeoutmsg` string.
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
```jldoctest
|
- Uses Julia's `@task`, `schedule`, and `Timer` to implement non-blocking
|
||||||
|
execution with interruption via `Base.throwto`.
|
||||||
|
- Errors with `InterruptException` if the function exceeds the timeout.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
julia> function testfunc(x)
|
julia> function testfunc(x)
|
||||||
sleep(x)
|
sleep(x)
|
||||||
return "task done"
|
return "task done"
|
||||||
end
|
end
|
||||||
julia> result = timeout(testfunc, 10; fargs=20)
|
julia> result = timeout(testfunc, 10; fargs=20)
|
||||||
"task timed out"
|
"task timed out"
|
||||||
julia> result = timeout(testfunc, 20; fargs=10)
|
julia> result = timeout(testfunc, 20; fargs=10)
|
||||||
"task done"
|
"task done"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function timeout(f::Function, timeoutwindow::Integer; fargs=nothing, timeoutmsg="task timed out")
|
function timeout(f::Function, timeoutwindow::Integer; fargs=nothing, timeoutmsg="task timed out")
|
||||||
tsk = @task f(fargs)
|
tsk = @task f(fargs)
|
||||||
@@ -340,23 +361,26 @@ end
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
""" Convert a dataframe into CSV.
|
""" Converts a DataFrame to a CSV string representation using the CSV.jl package.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `df::DataFrame`
|
- `df::DataFrame`
|
||||||
A connection object to Postgres database
|
The DataFrame to convert to CSV format.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- `result::String`
|
- `String`: The DataFrame contents as a CSV-formatted string.
|
||||||
|
|
||||||
# Example
|
# Notes
|
||||||
|
- Uses `CSV.write` with an `IOBuffer` to capture the output as a string.
|
||||||
|
- The returned string contains the full CSV representation including headers.
|
||||||
|
|
||||||
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> using DataFrames, GeneralUtils
|
julia> using DataFrames, GeneralUtils
|
||||||
julia> df = DataFrame(A=1:3, B=5:7, fixed=1)
|
julia> df = DataFrame(A=1:3, B=5:7, fixed=1)
|
||||||
julia> result = GeneralUtils.dataframeToCSV(df)
|
julia> dataframeToCSV(df)
|
||||||
|
"1,5,1\n2,6,1\n3,7,1\n"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function dataframeToCSV(df::DataFrame)
|
function dataframeToCSV(df::DataFrame)
|
||||||
# Create an IOBuffer to capture the output
|
# Create an IOBuffer to capture the output
|
||||||
@@ -426,8 +450,6 @@ end
|
|||||||
3 => [Dict("a"=>7), Dict("a"=>8), Dict("a"=>9)]
|
3 => [Dict("a"=>7), Dict("a"=>8), Dict("a"=>9)]
|
||||||
4 => [Dict("a"=>10)]
|
4 => [Dict("a"=>10)]
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function disintegrate_vectorDict(data::Vector, partsize::Integer
|
function disintegrate_vectorDict(data::Vector, partsize::Integer
|
||||||
)
|
)
|
||||||
@@ -473,8 +495,6 @@ end
|
|||||||
julia> getDataFrameValue(df[1, :], :name)
|
julia> getDataFrameValue(df[1, :], :name)
|
||||||
"Alice"
|
"Alice"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
getDataFrameValue(row::DataFrameRow, key::Symbol) = row.:($key)
|
getDataFrameValue(row::DataFrameRow, key::Symbol) = row.:($key)
|
||||||
|
|
||||||
@@ -543,8 +563,6 @@ end
|
|||||||
julia> dfToString(df)
|
julia> dfToString(df)
|
||||||
"1) name: Alice, age: 25\n2) name: Bob, age: 30"
|
"1) name: Alice, age: 25\n2) name: Bob, age: 30"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function dfToString(df::DataFrame)
|
function dfToString(df::DataFrame)
|
||||||
dfstr = ""
|
dfstr = ""
|
||||||
@@ -583,8 +601,6 @@ end
|
|||||||
"{\"name\":\"Alice\",\"age\":25}"
|
"{\"name\":\"Alice\",\"age\":25}"
|
||||||
"{\"name\":\"Bob\",\"age\":30}"
|
"{\"name\":\"Bob\",\"age\":30}"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function dataframe_to_json_list(df::DataFrame)::Vector{String}
|
function dataframe_to_json_list(df::DataFrame)::Vector{String}
|
||||||
json_list = []
|
json_list = []
|
||||||
@@ -618,8 +634,6 @@ end
|
|||||||
julia> dict_to_string(od)
|
julia> dict_to_string(od)
|
||||||
"1) name: Alice, 2) age: 25"
|
"1) name: Alice, 2) age: 25"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Signature
|
|
||||||
"""
|
"""
|
||||||
function dictToString(od::T) where {T<:AbstractDict}
|
function dictToString(od::T) where {T<:AbstractDict}
|
||||||
items = []
|
items = []
|
||||||
@@ -733,27 +747,39 @@ end
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
""" Remove French accents from the given text.
|
||||||
remove_french_accents(text::String) -> String
|
|
||||||
|
|
||||||
Remove French accents from the given text.
|
The function replaces accented French characters with their non-accented
|
||||||
|
counterparts using a dictionary mapping. Supported accented characters
|
||||||
|
include: à, â, ä, á, é, è, ê, ë, î, ï, í, ñ, ô, ö, ò, ó, ù, û, ü, ÿ, ç,
|
||||||
|
and their uppercase variants. The apostrophe character `’` is removed
|
||||||
|
completely.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `text::String` The input string containing French accents.
|
- `text::AbstractString`
|
||||||
|
The input string containing French accented characters.
|
||||||
|
|
||||||
# Returns
|
# Return
|
||||||
- `String` The input string with all French accents removed.
|
- `AbstractString`: A new string with all French accents replaced by their
|
||||||
|
non-accented equivalents.
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
- The function creates a character list and replaces each accented character
|
||||||
|
according to an internal dictionary mapping.
|
||||||
|
- Does **not** mutate the input; it allocates a new string.
|
||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
julia> remove_french_accents("Café")
|
julia> using GeneralUtils
|
||||||
"Cafe"
|
julia> remove_french_accents("Café")
|
||||||
|
"Cafe"
|
||||||
|
|
||||||
julia> remove_french_accents("L'été est beau.")
|
julia> remove_french_accents("L'été est beau.")
|
||||||
"L'ete est beau."
|
"L'ete est beau."
|
||||||
```
|
|
||||||
|
|
||||||
# Signature
|
julia> remove_french_accents("Noël, naïve, François")
|
||||||
|
"Noel, naive, Francois"
|
||||||
|
```
|
||||||
"""
|
"""
|
||||||
function remove_french_accents(text::AbstractString)::AbstractString
|
function remove_french_accents(text::AbstractString)::AbstractString
|
||||||
textcharlist = [i for i in text]
|
textcharlist = [i for i in text]
|
||||||
@@ -1113,9 +1139,38 @@ function nonRecentElementsIndex(vectorlength::Integer, n::Integer)
|
|||||||
return 1:(vectorlength-n)
|
return 1:(vectorlength-n)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
""" Remove specified substrings from text.
|
||||||
|
|
||||||
|
Removes all occurrences of each string in `removelist` from the input text
|
||||||
|
by repeatedly replacing them with empty strings until none remain.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `text::String`
|
||||||
|
The input string to modify.
|
||||||
|
- `removelist::Vector{String}`
|
||||||
|
A vector of substrings to remove from the text.
|
||||||
|
|
||||||
|
# Return
|
||||||
|
- `String`: The text with all specified substrings removed.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
```jldoctest
|
||||||
|
julia> using GeneralUtils
|
||||||
|
julia> removestring("hello world", ["l", " "])
|
||||||
|
"heoword"
|
||||||
|
julia> removestring("foo bar baz", ["bar", " "])
|
||||||
|
"foobaz"
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function removestring(text::String, removelist::Vector{String})::String
|
||||||
|
for i in removelist
|
||||||
|
while occursin(i, text)
|
||||||
|
text = replace(text, i => "")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
return string(text)
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user