update

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.12.7"
 manifest_format = "2.0"
-project_hash = "4450296e6bb4e5e245de62f43b73b263b215aa5d"
+project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044"
 
 [[deps.AWS]]
 deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"]
@@ -306,7 +306,7 @@ version = "1.2.0"
 deps = ["AWS", "AWSS3", "CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "Graphs", "HTTP", "JSON", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"]
 path = "."
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.6.0"
+version = "0.6.1"
 
 [[deps.GitHub]]
 deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"]

Project.toml

@@ -16,11 +16,13 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
 NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
+URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
@@ -31,5 +33,7 @@ HTTP = "2.5.0 - 2.9.9"
 JSON = "1.3.0 - 1.9.9"
 LibPQ = "1.18.0"
 NATS = "0.1.0"
+OrderedCollections = "2.0.1"
 Revise = "3.13.2"
 StringDistances = "1.0.0"
+URIs = "1.7.0"

etc.jl

@@ -0,0 +1,271 @@
+using SHA
+using Dates
+using URIs
+using OrderedCollections
+using AWS 
+@service S3
+using AWSS3
+
+
+# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
+# this file use local AWS config
+
+""" Example
+storage = GeneralUtils.GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+  "sommpanion-s3"
+)
+
+# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
+GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
+keys = GeneralUtils.list_files(storage)
+
+println("Read back: ", data)
+println("Bucket keys: ", keys)
+
+# test with curl
+curl -v \
+  -H 'Host: sommpanion-s3.s3-web.yiem.cc' \
+  http://192.168.88.106:3902/users-1002.json
+
+# test with API
+data = String(GeneralUtils.get_file(storage, "users-1005.json"))
+
+# test with a browser
+https://s3-web.mydomain.com/my_bucket_name/users-1005.json
+
+"""
+
+# ===================================================================
+# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
+# ===================================================================
+
+struct SimpleCredentials
+    access_key_id::String
+    secret_key::String
+    token::String
+end
+
+struct GarageConfig <: AWS.AbstractAWSConfig
+    endpoint::String
+    region::String
+    credentials::SimpleCredentials
+end
+
+# Required extensions for AWS.jl pipeline
+AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
+AWS.credentials(c::SimpleCredentials) = c
+AWS.check_credentials(c::SimpleCredentials) = c
+AWS.region(aws::GarageConfig) = aws.region
+AWS.credentials(aws::GarageConfig) = aws.credentials
+
+""" Generate the base URL for a Garage S3 service request.
+
+# Arguments
+- `aws::GarageConfig`
+  The Garage configuration containing the endpoint.
+- `service::String`
+  The AWS service name (e.g., `"s3"`).
+- `region::String`
+  The AWS region string.
+
+# Return
+- `String`: The stripped endpoint URL (trailing slashes removed).
+"""
+function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
+    return strip(aws.endpoint, '/')
+end
+
+""" Generate the full service URL including the resource path for a Garage S3 request.
+
+# Arguments
+- `aws::GarageConfig`
+  The Garage configuration containing the endpoint.
+- `service::String`
+  The AWS service name (e.g., `"s3"`).
+- `resource::String`
+  The S3 resource path (e.g., `"/bucket/key"`).
+
+# Return
+- `String`: The full URL combining the endpoint and resource path.
+"""
+function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
+    endpoint = strip(aws.endpoint, '/')
+    resource_path = startswith(resource, '/') ? resource : "/" * resource
+    return string(endpoint, resource_path)
+end
+
+# ===================================================================
+# 2. Thread-Safe Storage Client Wrapper
+# ===================================================================
+
+""" Create a thread-safe Garage S3 storage client.
+
+# Arguments
+- `endpoint::String`
+  The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
+- `key::String`
+  The Garage access key ID (created via garage-ui).
+- `secret::String`
+  The secret key corresponding to the access key.
+- `bucket::String`
+  The name of the S3 bucket to use.
+
+# Keyword Arguments
+- `region::String = "garage"`
+  The region identifier for the Garage instance.
+
+# Return
+- `GarageStorage`: A thread-safe storage client instance.
+
+# Examples
+```jldoctest
+julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
+GarageStorage(GarageConfig(...), "sommpanion-s3")
+```
+"""
+struct GarageStorage
+    config::GarageConfig
+    bucket::String
+end
+function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
+    creds = SimpleCredentials(key, secret, "")
+    config = GarageConfig(endpoint, region, creds)
+    return GarageStorage(config, bucket)
+end
+
+
+# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl
+function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}})
+    return SHA.hmac_sha256(key, msg)
+end
+
+function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}})
+    return SHA.hmac_sha256(Vector{UInt8}(key), msg)
+end
+
+# 2. AWSS3 overload targeting your custom Garage endpoint
+function AWSS3._s3_sign_url_v4(
+    aws::GarageConfig,
+    bucket::String,
+    path::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    protocol::String="https",
+)
+    path_escaped = URIs.escapepath("/$bucket/$path")
+
+    now_datetime = now(Dates.UTC)
+    datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
+    date_stamp = Dates.format(now_datetime, "YYYYmmdd")
+
+    service = "s3"
+    scheme = "AWS4"
+    algorithm = "HMAC-SHA256"
+    terminator = "aws4_request"
+
+    # Robust host & port extraction (prevents trailing colon bugs like 'domain.com:')
+    parsed_uri = URIs.URI(aws.endpoint)
+    has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0
+    host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host
+    effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme
+
+    scope = "$date_stamp/$(aws.region)/$service/$terminator"
+
+    headers = OrderedDict{String,String}("Host" => host)
+    sort!(headers; by=name -> lowercase(name))
+    canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
+
+    query = OrderedDict{String,String}(
+        "X-Amz-Expires" => string(seconds),
+        "X-Amz-Algorithm" => "$scheme-$algorithm",
+        "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope",
+        "X-Amz-Date" => datetime_stamp,
+        "X-Amz-SignedHeaders" => canonical_header_names,
+    )
+
+    if !isempty(aws.credentials.token)
+        query["X-Amz-Security-Token"] = aws.credentials.token
+    end
+
+    sort!(query; by=name -> lowercase(name))
+
+    canonical_headers = join(
+        map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers))
+    )
+
+    canonical_request = string(
+        "$verb\n",
+        "$path_escaped\n",
+        "$(URIs.escapeuri(query))\n",
+        "$canonical_headers\n",
+        "$canonical_header_names\n",
+        "UNSIGNED-PAYLOAD",
+    )
+
+    hashed_canonical_request = bytes2hex(sha256(canonical_request))
+
+    string_to_sign = string(
+        "$scheme-$algorithm\n",
+        "$datetime_stamp\n",
+        "$scope\n",
+        hashed_canonical_request,
+    )
+
+    # AWS Signature V4 Key Derivation Chain
+    key_secret = string(scheme, aws.credentials.secret_key)
+    key_date = _hmac_sha256(key_secret, date_stamp)
+    key_region = _hmac_sha256(key_date, aws.region)
+    key_service = _hmac_sha256(key_region, service)
+    key_signing = _hmac_sha256(key_service, terminator)
+    signature = _hmac_sha256(key_signing, string_to_sign)
+
+    query["X-Amz-Signature"] = bytes2hex(signature)
+
+    return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query))
+end
+
+
+
+"""
+    generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
+
+Generates direct public HTTP GET URLs via Garage's s3-web interface.
+Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint.
+"""
+function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
+    clean_endpoint = strip(web_endpoint, '/')
+    clean_key = lstrip(key, '/')
+
+    # s3-web URL format: https://<web-domain>/<bucket>/<key>
+    download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)"
+
+    return (
+        download_url = download_url,
+        # DELETE requires s3-api (s3-web does not support HTTP DELETE)
+        delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE")
+    )
+end
+
+
+
+
+
+# 1. Initialize Garage configuration pointing to S3 API domain
+storage = GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+  "sommpanion-s3"
+)
+
+# 2. Upload microservice file payload
+put_file(storage, "job-9042-data.bin", payload_bytes)
+
+# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes)
+urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900)
+
+# Pass `urls.download_url` and `urls.delete_url` to the downstream service

src/garageS3.jl

@@ -1,42 +1,43 @@
 module garageS3
 
 export
-    GarageStorage,
+    GarageStorage, put_file, get_file, list_files, delete_file,
-    put_file,
+    set_lifecycle_expiration, get_lifecycle, delete_lifecycle
-    get_file,
-    list_files,
-    delete_file
 
-using AWS, AWSS3
+using AWS 
+@service S3
+using AWSS3
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 # Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
 # this file use local AWS config
 
 """ Example
-const storage = GarageStorage(
+storage = GeneralUtils.GarageStorage(
-    "https://s3-api.yiem.cc",
+  "https://s3-api.yiem.cc",
-    "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
-    "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
-    "sommpanion-s3"
+  "sommpanion-s3"
 )
 
 # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
-put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
+# use UUID as filename because GarageS3 requires unique filename for each uploaded object
-keys = list_files(storage)
+urls = GeneralUtils.put_file(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
-data = String(get_file(storage, "users-1004.json"))
+key = GeneralUtils.list_files(storage)
 
 println("Read back: ", data)
-println("Bucket keys: ", keys)
+println("Bucket keys: ", key)
 
 # test with curl
 curl -v \
   -H 'Host: sommpanion-s3.s3-web.yiem.cc' \
   http://192.168.88.106:3902/users-1002.json
 
+# test with API
+data = String(GeneralUtils.get_file(storage, key[1]))
 
-# test with a browser
+# test with a browser (urls.web)
-https://s3-web.mydomain.com/my_bucket_name/users-1002.json
+https://s3-web.mydomain.com/my_bucket_name/users-1005.json
 
 """
 
@@ -103,10 +104,6 @@ end
 # 2. Thread-Safe Storage Client Wrapper
 # ===================================================================
 
-struct GarageStorage
-    config::GarageConfig
-    bucket::String
-end
 """ Create a thread-safe Garage S3 storage client.
 
 # Arguments
@@ -128,16 +125,72 @@ end
 
 # Examples
 ```jldoctest
-julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "sommpanion-s3")
+julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
 GarageStorage(GarageConfig(...), "sommpanion-s3")
 ```
 """
+struct GarageStorage
+    config::GarageConfig
+    bucket::String
+end
 function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
     creds = SimpleCredentials(key, secret, "")
     config = GarageConfig(endpoint, region, creds)
     return GarageStorage(config, bucket)
 end
 
+""" Upload an object to the Garage S3 bucket (callable struct).
+
+# Arguments
+- `key::String`
+  The object key (name) in the bucket. Must not contain slashes (`/`).
+- `data::Union{String, Vector{UInt8}}`
+  The data to upload.
+
+# Return
+- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs if the upload succeeds.
+- `nothing` if the key contains slashes (upload aborted with warning).
+
+# Notes
+- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
+  convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
+
+# Examples
+```jldoctest
+julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
+julia> put_file1 = put_file(storage1)
+julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}")
+Successfully uploaded:  test.json
+julia> downloadUrl.api
+"https://s3-api.yiem.cc/sommpanion-s3/test.json"
+julia> downloadUrl.web
+"https://s3-web.yiem.cc/sommpanion-s3/test.json"
+```
+"""
+struct put_file
+  storage::GarageStorage
+end
+function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}}
+  )::Union{NamedTuple{(:api, :web), Tuple{String, String}}, Nothing}
+  # Check if the key contains a slash (virtual folder character)
+  if occursin('/', key)
+      @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
+      return nothing
+  end
+  
+  # Proceed if the key is flat
+  AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
+  println("Successfully uploaded: ", key)
+
+  # object url
+  api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
+
+  # file download URL using browser
+  web = replace(api, "s3-api" => "s3-web")
+
+  return (api=api, web=web)
+end
+
 """ Upload an object to the Garage S3 bucket.
 
 # Arguments
@@ -149,8 +202,8 @@ end
   The data to upload.
 
 # Return
-- `Nothing` if the key contains slashes (upload aborted with warning).
+- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs if the upload succeeds.
-- Prints a success message if the upload completes.
+- `nothing` if the key contains slashes (upload aborted with warning).
 
 # Notes
 - Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
@@ -158,20 +211,33 @@ end
 
 # Examples
 ```julia
-julia> put_file(storage, "test.json", "{\"status\": \"active\"}")
+julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}")
 Successfully uploaded:  test.json
+julia> result.api
+"https://s3-api.yiem.cc/sommpanion-s3/test.json"
+julia> result.web
+"https://s3-web.yiem.cc/sommpanion-s3/test.json"
 ```
 """
-function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})
+function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
-    # Check if the key contains a slash (virtual folder character)
+  )::Union{NamedTuple{(:api, :web), Tuple{String, String}}, Nothing}
-    if occursin('/', key)
+  # Check if the key contains a slash (virtual folder character)
-        @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
+  if occursin('/', key)
-        return nothing
+      @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
-    end
+      return nothing
-    
+  end
-    # Proceed if the key is flat
+  
-    s3_put(storage.config, storage.bucket, key, data)
+  # Proceed if the key is flat
-    println("Successfully uploaded: ", key)
+  AWSS3.s3_put(storage.config, storage.bucket, key, data)
+  println("Successfully uploaded: ", key)
+
+  # object url
+  api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
+
+  # file download URL using browser
+  web = replace(api, "s3-api" => "s3-web")
+
+  return (api=api, web=web)
 end
 
 """ Download an object from the Garage S3 bucket.
@@ -191,8 +257,12 @@ julia> data = String(get_file(storage, "test.json"))
 "\"{\\\"status\\\": \\\"active\\\"}\""
 ```
 """
-function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
+function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
-    return s3_get(storage.config, storage.bucket, key)
+    try
+        return AWSS3.s3_get(storage.config, storage.bucket, key)
+    catch
+        return nothing
+    end
 end
 
 """ List all object keys in the Garage S3 bucket.
@@ -215,7 +285,7 @@ julia> keys = list_files(storage)
 """
 function list_files(storage::GarageStorage)
     # Approach 2: s3_list_objects returns a Vector of Dicts with object details
-    objects = s3_list_objects(storage.config, storage.bucket)
+    objects = AWSS3.s3_list_objects(storage.config, storage.bucket)
     return [obj["Key"] for obj in objects]
 end
 
@@ -233,7 +303,7 @@ julia> delete_file(storage, "test.json")
 ```
 """
 function delete_file(storage::GarageStorage, key::String)
-    s3_delete(storage.config, storage.bucket, key)
+    AWSS3.s3_delete(storage.config, storage.bucket, key)
 end
 
 
@@ -244,11 +314,4 @@ end
 
 
 
-
-
-
-
-
-
-
 end # module GarageS3