update

2026-09-05 10:27:31 +07:00 · 2026-09-05 10:00:36 +07:00 · 2026-09-04 14:05:36 +07:00 · 2026-09-04 13:58:28 +07:00 · 2026-09-01 14:35:57 +07:00 · 2026-09-01 12:21:17 +07:00
11 changed files with 718 additions and 44 deletions
@@ -0,0 +1,27 @@
+# 1. Garage Web Admin UI
+garage-ui.yiem.cc {
+    reverse_proxy 192.168.88.106:3909
+}
+
+# 2. Garage S3 API (Path-Style & Single Domain)
+s3-api.yiem.cc {
+    reverse_proxy 192.168.88.106:3900 {
+        header_up Host {host}
+    }
+}
+
+# 3. Garage S3 WEB (Path-Style & Single Domain)
+s3-web.yiem.cc {
+    # /bucket -> /bucket/
+    @bucket_root path_regexp bucket_root ^/([^/]+)$
+    redir @bucket_root /{http.regexp.bucket_root.1}/ 301
+
+    # /bucket/... -> Garage
+    @bucket path_regexp bucket ^/([^/]+)(/.*)?$
+
+    rewrite @bucket {http.regexp.bucket.2}
+
+    reverse_proxy 192.168.88.106:3902 {
+        header_up Host {http.regexp.bucket.1}.s3-web.yiem.cc
+    }
+}
@@ -2,7 +2,7 @@

 julia_version = "1.12.7"
 manifest_format = "2.0"
-project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044"
+project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2"

 [[deps.AWS]]
 deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"]
@@ -1,11 +1,12 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.6.2"
-authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
+version = "0.6.9"
+authors = ["tonaerospace <tonaerospace.etc@gmail.com"]

 [deps]
 AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
 AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
@@ -15,6 +16,7 @@ Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
+MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
 NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -28,10 +30,12 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [compat]
 AWS = "1.99.0"
 AWSS3 = "0.11.6"
+Base64 = "1.11.0"
 Graphs = "1.14.0"
 HTTP = "2.5.0 - 2.9.9"
 JSON = "1.3.0 - 1.9.9"
 LibPQ = "1.18.0"
+MbedTLS = "1.1.10"
 NATS = "0.1.0"
 OrderedCollections = "2.0.1"
 Revise = "3.13.2"
@@ -0,0 +1,52 @@
+# ==========================================
+# Garage Storage Engine Configuration
+# ==========================================
+
+# Core storage directories
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+
+# LMDB is recommended for production performance
+db_engine = "lmdb"
+metadata_auto_snapshot_interval = "6h"
+
+# Multi-node ready setup:
+# Set to 1 for initial single node. Update to 3 when adding Node 2 and Node 3 later.
+replication_factor = 1
+compression_level = 2
+
+# ==========================================
+# Network & Node Identity Configuration
+# ==========================================
+
+# Bind to all interfaces for host network mode
+rpc_bind_addr = "[::]:3901"
+
+# IMPORTANT: Set to Node 1's publicly reachable IP or DNS name (e.g., 192.168.1.10:3901 or 203.0.113.10:3901)
+rpc_public_addr = "192.168.88.106:3901"
+
+# Generate with: openssl rand -hex 32 (MUST BE IDENTICAL ON ALL NODES IN CLUSTER)
+rpc_secret = "3cf7a4d1da5d7b024a06db7fe6d73e2c160c6de640fda69b0b964d98678155a7"
+
+# ==========================================
+# Service APIs
+# ==========================================
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "0.0.0.0:3900"
+root_domain = ".s3-api.yiem.cc"
+
+[s3_web]
+bind_addr = "0.0.0.0:3902"
+root_domain = ".s3-web.yiem.cc"
+index = "index.html"
+
+[k2v_api]
+api_bind_addr = "0.0.0.0:3904"
+
+[admin]
+api_bind_addr = "0.0.0.0:3903"
+# Generate with: openssl rand -hex 16
+admin_token = "fd92f47a7c1ce8b072b06bb19dbc3fef"
+metrics_token = "27d29618cc436a23ec7d842392fdf74b"
@@ -0,0 +1,48 @@
+services:
+  garage:
+    image: dxflrs/garage:v2.3.0
+    container_name: garage
+    restart: unless-stopped
+    network_mode: "host"
+    volumes:
+      - ./garage.toml:/etc/garage.toml:ro
+      - garage_meta:/var/lib/garage/meta
+      - garage_data:/var/lib/garage/data
+    environment:
+      - RUST_LOG=garage=trace
+    healthcheck:
+      test: ["CMD", "/garage", "status"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+  webui:
+    image: khairul169/garage-webui:latest
+    container_name: garage-webui
+    restart: unless-stopped
+    network_mode: "host"
+    volumes:
+      - ./garage.toml:/etc/garage.toml:ro
+    environment:
+      API_BASE_URL: "http://127.0.0.1:3903"
+      S3_ENDPOINT_URL: "https://s3-api.yiem.cc"
+      # Format: "username:bcrypt_hash" (generate via: htpasswd -nbBC 10 "admin" "yourpassword")
+      AUTH_USER_PASS: "admin:$$2y$$10$$fhEucJHW8N5p8NL7Gv4Bi.FZVngbq/zatVDCjl.V.xNtj5q1PNUZW"
+    depends_on:
+      garage:
+        condition: service_healthy
+
+volumes:
+  garage_meta:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ./data/meta
+  garage_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ./data/storage
@@ -2,12 +2,18 @@ module garageS3

 export
    LifecycleExpiration, build_lifecycle_xml,
-    GarageStorage, put_file, get_file, list_files, delete_file,
-    set_lifecycle_expiration, get_lifecycle, delete_lifecycle
+    GarageStorage, s3PutFile, s3GetFile, s3ListFiles, s3DeleteFile,
+    s3SetLifecycleExpiration, s3GetLifecycle, s3DeleteLifecycle,
+    s3SignUrl, s3SignUrlWeb

 using AWS 
@service S3
 using AWSS3
+using Dates: Dates
+using URIs: URIs
+using OrderedCollections: OrderedDict
+using MbedTLS: digest, MD_SHA256
+using Base64: bytes2hex
 # ---------------------------------------------- 100 --------------------------------------------- #

 # Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
@@ -18,13 +24,13 @@ storage1 = GeneralUtils.GarageStorage(
  "https://s3-api.yiem.cc",
  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
-  "testbucket"
+  "sommpanion"
 )

 # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
 # use UUID as filename because GarageS3 requires unique filename for each uploaded object
-urls = GeneralUtils.put_file(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
-key = GeneralUtils.list_files(storage)
+urls = GeneralUtils.s3PutFile(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
+key = GeneralUtils.s3ListFiles(storage)

 println("Read back: ", data)
 println("Bucket keys: ", key)
@@ -35,7 +41,7 @@ curl -v \
  http://192.168.88.106:3902/users-1002.json

 # test with API
-data = String(GeneralUtils.get_file(storage, key[1]))
+data = String(GeneralUtils.s3GetFile(storage, key[1]))

 # test with a browser (urls.web)
 https://s3-web.mydomain.com/my_bucket_name/users-1005.json
@@ -46,13 +52,13 @@ https://s3-web.mydomain.com/my_bucket_name/users-1005.json
 # 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
 # ===================================================================

-struct SimpleCredentials
+mutable struct SimpleCredentials
    access_key_id::String
    secret_key::String
    token::String
 end

-struct GarageConfig <: AWS.AbstractAWSConfig
+mutable struct GarageConfig <: AWS.AbstractAWSConfig
    endpoint::String
    region::String
    credentials::SimpleCredentials
@@ -113,7 +119,7 @@ julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff948
 GarageStorage(GarageConfig(...), "sommpanion-s3")
 ```
 """
-struct GarageStorage
+mutable struct GarageStorage
    config::GarageConfig
    bucket::String
 end
@@ -137,8 +143,8 @@ end
 # Examples
 ```jldoctest
 julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
-julia> put_file1 = put_file(storage1)
-julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}")
+julia> s3PutFile1 = s3PutFile(storage1)
+julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}")
 Successfully uploaded:  test.json
 julia> downloadUrl.api
 "https://s3-api.yiem.cc/sommpanion-s3/test.json"
@@ -146,10 +152,10 @@ julia> downloadUrl.web
 "https://s3-web.yiem.cc/sommpanion-s3/test.json"
 ```
 """
-struct put_file
+mutable struct s3PutFile
  storage::GarageStorage
 end
-function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}}
+function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}}
  )::NamedTuple{(:api, :web), Tuple{String, String}}
  AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
  println("Successfully uploaded: ", key)
@@ -178,7 +184,7 @@ end

 # Examples
 ```julia
-julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}")
+julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}")
 Successfully uploaded:  test.json
 julia> result.api
 "https://s3-api.yiem.cc/sommpanion-s3/test.json"
@@ -186,7 +192,7 @@ julia> result.web
 "https://s3-web.yiem.cc/sommpanion-s3/test.json"
 ```
 """
-function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
+function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
  )::NamedTuple{(:api, :web), Tuple{String, String}}
  AWSS3.s3_put(storage.config, storage.bucket, key, data)
  println("Successfully uploaded: ", key)
@@ -213,11 +219,11 @@ end

 # Examples
 ```julia
-julia> data = String(get_file(storage, "test.json"))
+julia> data = String(s3GetFile(storage, "test.json"))
 "\"{\\\"status\\\": \\\"active\\\"}\""
 ```
 """
-function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
+function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
    try
        return AWSS3.s3_get(storage.config, storage.bucket, key)
    catch
@@ -236,14 +242,14 @@ end

 # Examples
 ```julia
-julia> keys = list_files(storage)
+julia> keys = s3ListFiles(storage)
 3-element Vector{String}:
 "test1.json"
 "test2.json"
 "users-1002.json"
 ```
 """
-function list_files(storage::GarageStorage)
+function s3ListFiles(storage::GarageStorage)
    # Use delimiter="" to get all individual keys (no grouping by "directory")
    objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="")
    return [obj["Key"] for obj in objects]
@@ -259,13 +265,196 @@ end

 # Examples
 ```julia
-julia> delete_file(storage, "test.json")
+julia> s3DeleteFile(storage, "test.json")
 ```
 """
-function delete_file(storage::GarageStorage, key::String)
+function s3DeleteFile(storage::GarageStorage, key::String)
    AWSS3.s3_delete(storage.config, storage.bucket, key)
 end

+"""
+    s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String
+
+Generate a pre-signed URL for an object in the Garage S3 bucket.
+
+A pre-signed URL grants temporary access without AWS credentials. Signature is computed
+using AWS Signature Version 4 (HMAC-SHA256).
+
+# Arguments
+- `storage::GarageStorage`: Storage client from `GarageStorage(endpoint, key, secret, bucket)`.
+- `key::String`: Object key. Supports nested paths like `"folder/subfolder/file.txt"`.
+- `seconds::Int`: URL validity in seconds. Default: 3600.
+
+# Keyword Arguments
+- `verb::String = "GET"`: HTTP method (`"GET"` downloads, `"PUT"` uploads).
+- `content_type::String = "application/octet-stream"`: Required `Content-Type` header.
+  For PUT, set to the file's MIME type (`"image/png"`, `"application/json"`, etc.).
+- `signature_version::String = "v4"`: Always use `"v4"` (v2 is deprecated).
+
+# Return
+`String`: The pre-signed URL.
+
+# Examples
+
+```julia
+using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile
+
+storage = GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+  "sommpanion"
+)
+
+# Upload — PUT with curl
+url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg")
+# curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' <url>
+
+# Download — GET with curl
+url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET")
+# curl <url>  → returns file content
+
+# Upload JSON
+url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json")
+# curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' <url>
+
+# Delete
+url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE")
+# curl -X DELETE url
+
+# Multiple uploads (chat app)
+urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300;
+                     verb="PUT", content_type="image/png")
+        for _ in 1:10]
+
+# Signature differs by verb and content_type
+url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET")
+url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain")
+```
+
+# Notes
+- URLs are valid until `seconds` expires; they can be used **multiple times** (not one-time).
+- Server clocks must be within ~5 minutes of the Garage server (UTC time in signature).
+- PUT uploads require the client's `Content-Type` to match `content_type`.
+- For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`.
+"""
+function s3SignUrl(
+    storage::GarageStorage,
+    key::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    signature_version::String="v4",
+)::String
+    config = storage.config
+    creds = config.credentials
+    bucket = storage.bucket
+    region = config.region
+    endpoint = strip(config.endpoint, '/')
+    scheme = "AWS4"
+    algorithm = "HMAC-SHA256"
+    terminator = "aws4_request"
+    service = "s3"
+
+    now_datetime = Dates.now(Dates.UTC)
+    datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
+    date_stamp = Dates.format(now_datetime, "YYYYmmdd")
+
+    scope = "$date_stamp/$region/$service/$terminator"
+
+    headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host)
+    if verb == "PUT"
+        headers["Content-Type"] = content_type
+    end
+    sort!(headers; by=name -> lowercase(name))
+    canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
+
+    query = OrderedDict{String,String}(
+        "X-Amz-Expires" => string(seconds),
+        "X-Amz-Algorithm" => "$scheme-$algorithm",
+        "X-Amz-Credential" => "$(creds.access_key_id)/$scope",
+        "X-Amz-Date" => datetime_stamp,
+        "X-Amz-SignedHeaders" => canonical_header_names,
+    )
+    if !isempty(creds.token)
+        query["X-Amz-Security-Token"] = creds.token
+    end
+
+    canonical_headers = join(
+        map(
+            header -> "$(lowercase(header.first)):$(lowercase(header.second))\n",
+            collect(headers),
+        ),
+    )
+
+    resource_path = "/$bucket/$key"
+
+    # Build canonical query string: sort by key=value pair lexicographically,
+    # URL-encoding each key and value (matching Garage's uri_encode)
+    sorted_q = sort(
+        [URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)],
+        by=lowercase,
+    )
+    canonical_query = join(sorted_q, "&")
+
+    canonical_request = string(
+        "$verb\n",
+        resource_path,
+        "\n",
+        canonical_query,
+        "\n",
+        canonical_headers,
+        "\n",
+        canonical_header_names,
+        "\n",
+        "UNSIGNED-PAYLOAD",
+    )
+
+    string_to_sign = string(
+        "$scheme-$algorithm\n",
+        "$datetime_stamp\n",
+        "$scope\n",
+        bytes2hex(digest(MD_SHA256, canonical_request)),
+    )
+
+    key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key))
+    key_date = digest(MD_SHA256, date_stamp, key_secret_bytes)
+    key_region = digest(MD_SHA256, region, key_date)
+    key_service = digest(MD_SHA256, service, key_region)
+    key_signing = digest(MD_SHA256, terminator, key_service)
+    signature = digest(MD_SHA256, string_to_sign, key_signing)
+
+    query["X-Amz-Signature"] = bytes2hex(signature)
+
+    query_string = URIs.escapeuri(query)
+    return "$(endpoint)/$bucket/$key?$query_string"
+end
+
+""" Generate a pre-signed URL using the Garage S3 web endpoint (s3-web).
+
+Useful for direct browser access — URLs look like
+`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...`
+
+Arguments and keyword arguments are the same as `s3SignUrl`.
+"""
+function s3SignUrlWeb(
+    storage::GarageStorage,
+    key::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    signature_version::String="v4",
+)::String
+    web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web")
+    web_creds = SimpleCredentials(
+        storage.config.credentials.access_key_id,
+        storage.config.credentials.secret_key,
+        storage.config.credentials.token,
+    )
+    web_config = GarageConfig(web_endpoint, storage.config.region, web_creds)
+    web_storage = GarageStorage(web_config, storage.bucket)
+    return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version)
+end

 # ===================================================================
 # 3. Bucket Lifecycle Configuration
@@ -279,7 +468,7 @@ end
 - `enabled::Bool`: Whether the rule is active (default: `true`).
 - `id::String`: Unique identifier for the rule (default: auto-generated).
 """
-struct LifecycleExpiration
+mutable struct LifecycleExpiration
    prefix::String
    days::Int
    enabled::Bool
@@ -323,10 +512,10 @@ end
 # Example
 ```julia
 rule = LifecycleExpiration("uploads/", 30)
-set_lifecycle_expiration(storage, rule)
+s3SetLifecycleExpiration(storage, rule)
 ```
 """
-function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration)
+function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration)
    xml_body = build_lifecycle_xml(rule)
    args = Dict{String,Any}("body" => xml_body)
    return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config)
@@ -342,10 +531,10 @@ end

 # Example
 ```julia
-config = get_lifecycle(storage)
+config = s3GetLifecycle(storage)
 ```
 """
-function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
+function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
    try
        return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config)
    catch e
@@ -366,10 +555,10 @@ end

 # Example
 ```julia
-delete_lifecycle(storage)
+s3DeleteLifecycle(storage)
 ```
 """
-function delete_lifecycle(storage::GarageStorage)
+function s3DeleteLifecycle(storage::GarageStorage)
    return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config)
 end

@@ -4,12 +4,12 @@ export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, rep
        findMatchingDictKey, randstring, randstrings, timeout,
        dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString, 
        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
-        dictToString_numbering, extract_triple_backtick_text, 
+        dictToString_numbering, extract_triple_backtick_text, eventSink,
        countGivenWords, remove_french_accents, removestring,
        extractTextBetweenCharacter, extractTextBetweenString, 
        convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex

-using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, NATS, Dates

 # ---------------------------------------------- 100 --------------------------------------------- #

@@ -1173,6 +1173,106 @@ function removestring(text::String, removelist::Vector{String})::String
 end


+"""
+Create an event sink for logging messages and/or publishing to NATS or printing to console. File logging defaults to false.
+
+# Fields
+- `natsConn`: NATS connection, or `nothing` for file-only sink
+- `topic`: NATS topic to publish to, or `nothing` for file-only sink
+- `service_name`: Optional prefix added to log entries
+- `log_file`: Path to the log file
+- `max_log_entries`: Maximum number of entries to keep in the log file
+
+# Callable interface
+```julia
+    (es::eventSink)(msg::String; log=false, publish=true, console=true) -> Nothing
+```
+
+Three independent outputs — each defaults to `true` except `log`:
+- `console` (default `true`) → println to stdout
+- `publish` (default `true`) → publish to NATS topic (no-op if no connection)
+- `log` (default `false`) → append to `log_file`
+
+# Examples
+```julia
+using NATS, Dates
+
+conn = NATS.connect("nats.yiem.cc")
+
+# ——— With NATS (console + NATS by default) ———
+es = eventSink(natsConn=conn, topic="agent.events", service_name="agent1")
+es("starting up")                 # console + NATS
+es("logged"; log=true)            # console + file + NATS
+es("quiet"; console=false)        # NATS only
+es("file+nats"; log=true, console=false)  # file + NATS
+es("console-only"; publish=false, log=false)  # console only
+
+# ——— File-only sink (no NATS connection) ———
+es2 = eventSink(log_file="./log/eventSink2.md", service_name="worker")
+es2("ready")                      # console only
+es2("file-logged"; log=true)       # console + file
+es2("file-only"; log=true, console=false)  # file only
+es2("silent"; console=false)               # nothing (no NATS, log=false, console=false)
+
+NATS.drain(conn)
+```
+"""
+struct eventSink
+  natsConn::Union{NATS.Connection, Nothing}
+  topic::Union{String, Nothing}
+  service_name::String
+  log_file::String
+  max_log_entries::Int
+end
+eventSink(; natsConn=nothing, topic=nothing, service_name="", log_file="./log/eventSink.md", max_log_entries=100) =
+  eventSink(natsConn, topic, service_name, log_file, max_log_entries)
+function (es::eventSink)(msg::String; log=false, publish=true, console=true)
+    timestamp = Dates.format(now(), "yyyy-mm-ddTHH:MM:SS.sss")
+    svc_prefix = isempty(es.service_name) ? "" : "[$(es.service_name)] "
+    entry = "- [$timestamp] $svc_prefix$msg\n"
+    if console
+        println("[$timestamp] $svc_prefix$msg")
+    end
+    if log
+        log_dir = dirname(es.log_file)
+        mkpath(log_dir)
+        open(es.log_file, "a") do f
+            write(f, entry)
+        end
+        # Rotate after write — keep only last max_log_entries entries
+        content = read(es.log_file, String)
+        entries = split(content, r"- \[", keepempty=false)
+        if length(entries) > es.max_log_entries
+            rotate!(es)
+        end
+    end
+    if publish && es.topic !== nothing && es.natsConn !== nothing
+        try
+            NATS.publish(es.natsConn, es.topic, entry)
+        catch e
+            @warn "eventSink NATS publish failed" exception=e
+        end
+    end
+end
+
+"""
+    rotate!(es::eventSink) -> Nothing
+
+Rewrite the log file keeping only the last `max_log_entries` entries.
+Call after writing when the entry count reaches the limit.
+"""
+function rotate!(es::eventSink)
+    content = read(es.log_file, String)
+    entries = split(content, r"- \[", keepempty=false)
+    open(es.log_file, "w") do f
+        write(f, "- [")
+        for e in entries[2:end]
+            write(f, "- [", e)
+        end
+    end
+end
+
+



@@ -0,0 +1,130 @@
+curl -X PUT -H "Content-Type: application/json" \
+  -d '{"test": "data"}' \
+  "https://s3-api.yiem.cc/sommpanion/pcimage.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T103745Z&X-Amz-Expires=300&X-Amz-Security-Token=&X-Amz-SignedHeaders=host&X-Amz-Signature=0e195a48f824a2f8ffcb06461396cd6a8bddd5b61d93e9006f947ae10ecf525f"
+
+
+
+
+
+
+
+
+
+curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
+  -H 'Host: s3-api.yiem.cc' \
+  -H 'Content-Type: image/jpeg' \
+  "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T030315Z&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=1107a31f36f8b8ceb9663d69b9a7be7733df14d019e700e750944cbdad50df5d"
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+read
+./garage.toml
+./Caddyfile
+./src/garageS3.jl
+
+
+
+julia> url = GeneralUtils.s3_sign_url(storage1, "photo.jpg", 36000; verb="PUT", content_type="image/jpeg")
+"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T145004Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=92cd634dbb0f66db4b8b039ef5939c138203de260e8342fe27ef78979d90a824"
+
+---
+
+
+
+
+
+
+
+
+
+
+
+ton@server12vm-dev:~/docker-apps/sommpanion/etc/AWS.jl$ 
+curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
+  -H 'Host: s3-api.yiem.cc' \
+  -H 'Content-Type: image/jpeg' \
+  "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02"
+
+Note: Unnecessary use of -X or --request, PUT is already inferred.
+*   Trying 110.77.160.154:443...
+* Connected to s3-api.yiem.cc (110.77.160.154) port 443 (#0)
+* ALPN: offers h2,http/1.1
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+*  CAfile: /etc/ssl/certs/ca-certificates.crt
+*  CApath: /etc/ssl/certs
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=s3-api.yiem.cc
+*  start date: Aug 27 04:27:02 2026 GMT
+*  expire date: Nov 25 04:27:01 2026 GMT
+*  subjectAltName: host "s3-api.yiem.cc" matched cert's "s3-api.yiem.cc"
+*  issuer: C=US; O=Let's Encrypt; CN=YE2
+*  SSL certificate verify ok.
+* using HTTP/2
+* h2h3 [:method: PUT]
+* h2h3 [:path: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02]
+* h2h3 [:scheme: https]
+* h2h3 [:authority: s3-api.yiem.cc]
+* h2h3 [user-agent: curl/7.88.1]
+* h2h3 [accept: */*]
+* h2h3 [content-type: image/jpeg]
+* h2h3 [content-length: 439490]
+* Using Stream ID: 1 (easy handle 0x559a08824c80)
+> PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 HTTP/2
+> Host: s3-api.yiem.cc
+> user-agent: curl/7.88.1
+> accept: */*
+> content-type: image/jpeg
+> content-length: 439490
+> 
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+* We are completely uploaded and fine
+< HTTP/2 403 
+< access-control-allow-origin: *
+< alt-svc: h3=":443"; ma=2592000
+< content-type: application/xml
+< date: Fri, 04 Sep 2026 15:23:13 GMT
+< via: 1.1 Caddy
+< content-length: 190
+< 
+* Connection #0 to host s3-api.yiem.cc left intact
+<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message>Forbidden: Invalid signature</Message><Resource>/sommpanion/photo.jpg</Resource><Region>garage</Region></Error>
+
+2026-09-04T15:23:13.796037Z  INFO garage_api_common::generic_server: 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02
+2026-09-04T15:23:13.796077Z DEBUG garage_api_common::generic_server: Request { method: PUT, uri: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02, version: HTTP/1.1, headers: {"host": "s3-api.yiem.cc", "user-agent": "curl/7.88.1", "content-length": "439490", "accept": "*/*", "content-type": "image/jpeg", "via": "2.0 Caddy", "x-forwarded-for": "110.77.160.154", "x-forwarded-host": "s3-api.yiem.cc", "x-forwarded-proto": "https", "accept-encoding": "gzip"}, body: Body(Streaming) }
+2026-09-04T15:23:13.796181Z DEBUG garage_api_common::generic_server: Endpoint: PutObject
+2026-09-04T15:23:13.796461Z  INFO garage_api_common::generic_server: error 403 Forbidden, Forbidden: Invalid signature in response to 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -2,8 +2,7 @@ using Test
 using GeneralUtils
 using GeneralUtils.garageS3
 using AWS
-using Random: randstring
-using UUIDs: uuid4
+using URIs

@testset "garageS3.jl" begin

@@ -142,7 +141,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "testrunner-$(randstring(8)).json"
+        test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json"
        test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}"

        # Test put_file(storage, key, data) - successful upload
@@ -165,7 +164,7 @@ using UUIDs: uuid4
        @test test_key in keys

        # Test put_file callable struct
-        test_key2 = "testrunner-callable-$(randstring(8)).json"
+        test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json"
        callable_result = put_file(storage)(test_key2, "callable-data")
        @test callable_result !== nothing
        @test callable_result.api != ""
@@ -194,7 +193,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "single/slash/test-$(randstring(8)).json"
+        test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json"
        test_data = "{\"level\": 1, \"nested\": true}"

        # Upload nested key
@@ -227,7 +226,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "deep/nested/folder/subfolder/test-$(randstring(8)).json"
+        test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json"
        test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}"

        # Upload deeply nested key
@@ -259,7 +258,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "callable/nested/test-$(randstring(8)).json"
+        test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json"
        test_data = "{\"callable\": true, \"nested\": true}"

        uploader = put_file(storage)
@@ -281,7 +280,7 @@ using UUIDs: uuid4
            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
            "testbucket"
        )
-        result = get_file(storage, "nonexistent-key-$(randstring(8)).json")
+        result = get_file(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json")
        @test result === nothing
    end

@@ -293,7 +292,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "testrunner-binary-$(randstring(8)).bin"
+        test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin"
        binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f]

        result = put_file(storage, test_key, binary_data)
@@ -314,7 +313,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "testrunner-list-$(randstring(8)).txt"
+        test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt"
        put_file(storage, test_key, "list-test-data")

        keys = list_files(storage)
@@ -332,7 +331,7 @@ using UUIDs: uuid4
            "testbucket"
        )

-        test_key = "testrunner-url-$(randstring(8)).json"
+        test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json"
        result = put_file(storage, test_key, "url-test")

        @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/")
@@ -460,4 +459,129 @@ using UUIDs: uuid4
        delete_lifecycle(storage)
    end

+    # --- Pre-signed URL tests ---
+
+    @testset "s3_sign_url returns a string" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "test.json", 3600)
+        @test url isa String
+        @test !isempty(url)
+    end
+
+    @testset "s3_sign_url URL format contains correct endpoint, bucket, key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "mykey.json", 1800)
+        @test occursin("https://s3-api.yiem.cc", url)
+        @test occursin("/testbucket/mykey.json?", url)
+    end
+
+    @testset "s3_sign_url contains required query parameters" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "test.json", 3600)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url)
+        @test occursin("X-Amz-Expires=3600", url)
+        @test occursin("X-Amz-Credential=", url)
+        @test occursin("X-Amz-Date=", url)
+        @test occursin("X-Amz-SignedHeaders=", url)
+        @test occursin("X-Amz-Signature=", url)
+    end
+
+    @testset "s3_sign_url with custom expiry duration" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "test.json", 600)
+        @test occursin("X-Amz-Expires=600", url)
+    end
+
+    @testset "s3_sign_url with GET verb produces different signature than default" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url_get = s3_sign_url(storage, "test.json", 3600; verb="GET")
+        url_default = s3_sign_url(storage, "test.json", 3600)
+        # GET verb should produce a valid signature
+        @test occursin("X-Amz-Signature=", url_get)
+        @test occursin("X-Amz-Signature=", url_default)
+        # Both URLs should have the same structure but potentially different signatures
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get)
+    end
+
+    @testset "s3_sign_url with PUT verb and content_type" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url_put = s3_sign_url(storage, "upload.json", 300; verb="PUT", content_type="application/json")
+        url_get = s3_sign_url(storage, "upload.json", 300; verb="GET")
+        # PUT and GET should produce different signatures (different canonical request)
+        @test occursin("X-Amz-Signature=", url_put)
+        @test occursin("X-Amz-Signature=", url_get)
+    end
+
+    @testset "s3_sign_url with special characters in key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        test_key = "folder/sub folder/file with spaces.json"
+        url = s3_sign_url(storage, test_key, 3600)
+        @test occursin("testbucket/", url)
+    end
+
+    @testset "s3_sign_url with deeply nested key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        test_key = "a/b/c/d/e/deep-file.json"
+        url = s3_sign_url(storage, test_key, 3600)
+        @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url)
+    end
+
+    @testset "integration: pre-signed URL can download existing file" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}"
+        put_file(storage, test_key, test_data)
+
+        url = s3_sign_url(storage, test_key, 300)
+        @test occursin("https://s3-api.yiem.cc/testbucket/", url)
+
+        delete_file(storage, test_key)
+    end
+
 end
Author	SHA1	Message	Date
ton	599765d7ef	update	2026-09-05 10:27:31 +07:00
ton	9ad732c436	update	2026-09-05 10:00:36 +07:00
ton	25ca9b96a2	update	2026-09-04 14:05:36 +07:00
ton	2827d22616	update	2026-09-04 13:58:28 +07:00
ton	151e4e74cf	update	2026-09-01 14:35:57 +07:00
ton	42b66366ae	update	2026-09-01 12:21:17 +07:00
ton	b0761fae75	update	2026-09-01 12:21:02 +07:00
ton	f9e952e5ed	update	2026-09-01 12:06:41 +07:00
ton	397885511a	add eventSink	2026-09-01 12:03:50 +07:00
ton	ee8ade1189	update	2026-08-31 15:52:40 +07:00
ton	20c51cf39f	update	2026-08-30 10:46:39 +07:00
ton	f09a53b6c3	update	2026-08-29 19:25:04 +07:00
ton	c078948a7c	update	2026-08-29 19:12:14 +07:00
ton	3780eb41df	update	2026-08-29 18:47:45 +07:00
ton	c28b7ebf84	Merge pull request 'V0.6.2' (#14 ) from v0.6.2 into main Reviewed-on: #14	2026-08-29 02:58:21 +00:00