update

etc.jl

@@ -0,0 +1,271 @@
+using SHA
+using Dates
+using URIs
+using OrderedCollections
+using AWS 
+@service S3
+using AWSS3
+
+
+# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
+# this file use local AWS config
+
+""" Example
+storage = GeneralUtils.GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+  "sommpanion-s3"
+)
+
+# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
+GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
+keys = GeneralUtils.list_files(storage)
+
+println("Read back: ", data)
+println("Bucket keys: ", keys)
+
+# test with curl
+curl -v \
+  -H 'Host: sommpanion-s3.s3-web.yiem.cc' \
+  http://192.168.88.106:3902/users-1002.json
+
+# test with API
+data = String(GeneralUtils.get_file(storage, "users-1005.json"))
+
+# test with a browser
+https://s3-web.mydomain.com/my_bucket_name/users-1005.json
+
+"""
+
+# ===================================================================
+# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
+# ===================================================================
+
+struct SimpleCredentials
+    access_key_id::String
+    secret_key::String
+    token::String
+end
+
+struct GarageConfig <: AWS.AbstractAWSConfig
+    endpoint::String
+    region::String
+    credentials::SimpleCredentials
+end
+
+# Required extensions for AWS.jl pipeline
+AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
+AWS.credentials(c::SimpleCredentials) = c
+AWS.check_credentials(c::SimpleCredentials) = c
+AWS.region(aws::GarageConfig) = aws.region
+AWS.credentials(aws::GarageConfig) = aws.credentials
+
+""" Generate the base URL for a Garage S3 service request.
+
+# Arguments
+- `aws::GarageConfig`
+  The Garage configuration containing the endpoint.
+- `service::String`
+  The AWS service name (e.g., `"s3"`).
+- `region::String`
+  The AWS region string.
+
+# Return
+- `String`: The stripped endpoint URL (trailing slashes removed).
+"""
+function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
+    return strip(aws.endpoint, '/')
+end
+
+""" Generate the full service URL including the resource path for a Garage S3 request.
+
+# Arguments
+- `aws::GarageConfig`
+  The Garage configuration containing the endpoint.
+- `service::String`
+  The AWS service name (e.g., `"s3"`).
+- `resource::String`
+  The S3 resource path (e.g., `"/bucket/key"`).
+
+# Return
+- `String`: The full URL combining the endpoint and resource path.
+"""
+function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
+    endpoint = strip(aws.endpoint, '/')
+    resource_path = startswith(resource, '/') ? resource : "/" * resource
+    return string(endpoint, resource_path)
+end
+
+# ===================================================================
+# 2. Thread-Safe Storage Client Wrapper
+# ===================================================================
+
+""" Create a thread-safe Garage S3 storage client.
+
+# Arguments
+- `endpoint::String`
+  The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
+- `key::String`
+  The Garage access key ID (created via garage-ui).
+- `secret::String`
+  The secret key corresponding to the access key.
+- `bucket::String`
+  The name of the S3 bucket to use.
+
+# Keyword Arguments
+- `region::String = "garage"`
+  The region identifier for the Garage instance.
+
+# Return
+- `GarageStorage`: A thread-safe storage client instance.
+
+# Examples
+```jldoctest
+julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
+GarageStorage(GarageConfig(...), "sommpanion-s3")
+```
+"""
+struct GarageStorage
+    config::GarageConfig
+    bucket::String
+end
+function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
+    creds = SimpleCredentials(key, secret, "")
+    config = GarageConfig(endpoint, region, creds)
+    return GarageStorage(config, bucket)
+end
+
+
+# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl
+function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}})
+    return SHA.hmac_sha256(key, msg)
+end
+
+function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}})
+    return SHA.hmac_sha256(Vector{UInt8}(key), msg)
+end
+
+# 2. AWSS3 overload targeting your custom Garage endpoint
+function AWSS3._s3_sign_url_v4(
+    aws::GarageConfig,
+    bucket::String,
+    path::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    protocol::String="https",
+)
+    path_escaped = URIs.escapepath("/$bucket/$path")
+
+    now_datetime = now(Dates.UTC)
+    datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
+    date_stamp = Dates.format(now_datetime, "YYYYmmdd")
+
+    service = "s3"
+    scheme = "AWS4"
+    algorithm = "HMAC-SHA256"
+    terminator = "aws4_request"
+
+    # Robust host & port extraction (prevents trailing colon bugs like 'domain.com:')
+    parsed_uri = URIs.URI(aws.endpoint)
+    has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0
+    host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host
+    effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme
+
+    scope = "$date_stamp/$(aws.region)/$service/$terminator"
+
+    headers = OrderedDict{String,String}("Host" => host)
+    sort!(headers; by=name -> lowercase(name))
+    canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
+
+    query = OrderedDict{String,String}(
+        "X-Amz-Expires" => string(seconds),
+        "X-Amz-Algorithm" => "$scheme-$algorithm",
+        "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope",
+        "X-Amz-Date" => datetime_stamp,
+        "X-Amz-SignedHeaders" => canonical_header_names,
+    )
+
+    if !isempty(aws.credentials.token)
+        query["X-Amz-Security-Token"] = aws.credentials.token
+    end
+
+    sort!(query; by=name -> lowercase(name))
+
+    canonical_headers = join(
+        map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers))
+    )
+
+    canonical_request = string(
+        "$verb\n",
+        "$path_escaped\n",
+        "$(URIs.escapeuri(query))\n",
+        "$canonical_headers\n",
+        "$canonical_header_names\n",
+        "UNSIGNED-PAYLOAD",
+    )
+
+    hashed_canonical_request = bytes2hex(sha256(canonical_request))
+
+    string_to_sign = string(
+        "$scheme-$algorithm\n",
+        "$datetime_stamp\n",
+        "$scope\n",
+        hashed_canonical_request,
+    )
+
+    # AWS Signature V4 Key Derivation Chain
+    key_secret = string(scheme, aws.credentials.secret_key)
+    key_date = _hmac_sha256(key_secret, date_stamp)
+    key_region = _hmac_sha256(key_date, aws.region)
+    key_service = _hmac_sha256(key_region, service)
+    key_signing = _hmac_sha256(key_service, terminator)
+    signature = _hmac_sha256(key_signing, string_to_sign)
+
+    query["X-Amz-Signature"] = bytes2hex(signature)
+
+    return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query))
+end
+
+
+
+"""
+    generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
+
+Generates direct public HTTP GET URLs via Garage's s3-web interface.
+Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint.
+"""
+function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
+    clean_endpoint = strip(web_endpoint, '/')
+    clean_key = lstrip(key, '/')
+
+    # s3-web URL format: https://<web-domain>/<bucket>/<key>
+    download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)"
+
+    return (
+        download_url = download_url,
+        # DELETE requires s3-api (s3-web does not support HTTP DELETE)
+        delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE")
+    )
+end
+
+
+
+
+
+# 1. Initialize Garage configuration pointing to S3 API domain
+storage = GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+  "sommpanion-s3"
+)
+
+# 2. Upload microservice file payload
+put_file(storage, "job-9042-data.bin", payload_bytes)
+
+# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes)
+urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900)
+
+# Pass `urls.download_url` and `urls.delete_url` to the downstream service

src/garageS3.jl

@@ -21,7 +21,7 @@ storage = GeneralUtils.GarageStorage(
 )
 
 # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
-GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
+downloadUrl = GeneralUtils.put_file(storage, "users1.json", "{\"status\": \"active\"}")
 keys = GeneralUtils.list_files(storage)
 
 println("Read back: ", data)
@@ -147,7 +147,7 @@ end
   The data to upload.
 
 # Return
-- `String`: The file download URL if the upload succeeds.
+- `NamedTuple{(:api, :web), Tuple{String, String}}`: The file download URLs if the upload succeeds (`api` for API access, `web` for web access).
 - `nothing` if the key contains slashes (upload aborted with warning).
 
 # Notes
@@ -160,7 +160,7 @@ julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94
 julia> put_file1 = put_file(storage1)
 julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}")
 Successfully uploaded:  test.json
-"https://s3-api.yiem.cc/sommpanion-s3/test.json"
+(api = "https://s3-api.yiem.cc/sommpanion-s3/test.json", web = "https://s3-web.yiem.cc/sommpanion-s3/test.json")
 ```
 """
 struct put_file
@@ -177,10 +177,13 @@ function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}})::String
   AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
   println("Successfully uploaded: ", key)
 
-  # file download URL
+  # file download URL using API
-  objURL = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
+  api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
 
-  return objURL
+  # file download URL using browser
+  web = replace(api, "s3-api" => "s3-web")
+
+  return (api=api, web=web)
 end
 
 """ Upload an object to the Garage S3 bucket.
@@ -194,7 +197,7 @@ end
   The data to upload.
 
 # Return
-- `String`: The file download URL if the upload succeeds.
+- `NamedTuple{(:api, :web), Tuple{String, String}}`: The file download URLs if the upload succeeds (`api` for API access, `web` for web access).
 - `nothing` if the key contains slashes (upload aborted with warning).
 
 # Notes
@@ -205,6 +208,7 @@ end
 ```julia
 julia> put_file(storage, "test.json", "{\"status\": \"active\"}")
 Successfully uploaded:  test.json
+(api = "https://s3-api.yiem.cc/sommpanion-s3/test.json", web = "https://s3-web.yiem.cc/sommpanion-s3/test.json")
 ```
 """
 function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})::String
@@ -218,10 +222,13 @@ function put_file(storage::GarageStorage, key::String, data::Union{String, Vecto
   AWSS3.s3_put(storage.config, storage.bucket, key, data)
   println("Successfully uploaded: ", key)
 
-  # file download URL
+  # file download URL using API
-  objURL = "$(storage.config.endpoint)/$(storage.bucket)/$key"
+  api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
 
-  return objURL
+  # file download URL using browser
+  web = replace(api, "s3-api" => "s3-web")
+
+  return (api=api, web=web)
 end
 
 """ Download an object from the Garage S3 bucket.