From dee56492542a760ae91ddb8e6ca807b506e7b74d Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 28 Aug 2026 14:23:18 +0700 Subject: [PATCH] update --- etc.jl | 271 ++++++++++++++++++++++++++++++++++++++++++++++++ src/garageS3.jl | 27 +++-- 2 files changed, 288 insertions(+), 10 deletions(-) diff --git a/etc.jl b/etc.jl index e69de29..70bf716 100644 --- a/etc.jl +++ b/etc.jl @@ -0,0 +1,271 @@ +using SHA +using Dates +using URIs +using OrderedCollections +using AWS +@service S3 +using AWSS3 + + +# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key). +# this file use local AWS config + +""" Example +storage = GeneralUtils.GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key + "sommpanion-s3" +) + +# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) +GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}") +keys = GeneralUtils.list_files(storage) + +println("Read back: ", data) +println("Bucket keys: ", keys) + +# test with curl +curl -v \ + -H 'Host: sommpanion-s3.s3-web.yiem.cc' \ + http://192.168.88.106:3902/users-1002.json + +# test with API +data = String(GeneralUtils.get_file(storage, "users-1005.json")) + +# test with a browser +https://s3-web.mydomain.com/my_bucket_name/users-1005.json + +""" + +# =================================================================== +# 1. Custom GarageConfig Definition (Thread-Safe, No Global State) +# =================================================================== + +struct SimpleCredentials + access_key_id::String + secret_key::String + token::String +end + +struct GarageConfig <: AWS.AbstractAWSConfig + endpoint::String + region::String + credentials::SimpleCredentials +end + +# Required extensions for AWS.jl pipeline +AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c +AWS.credentials(c::SimpleCredentials) = c +AWS.check_credentials(c::SimpleCredentials) = c +AWS.region(aws::GarageConfig) = aws.region +AWS.credentials(aws::GarageConfig) = aws.credentials + +""" Generate the base URL for a Garage S3 service request. + +# Arguments +- `aws::GarageConfig` + The Garage configuration containing the endpoint. +- `service::String` + The AWS service name (e.g., `"s3"`). +- `region::String` + The AWS region string. + +# Return +- `String`: The stripped endpoint URL (trailing slashes removed). +""" +function AWS.generate_service_url(aws::GarageConfig, service::String, region::String) + return strip(aws.endpoint, '/') +end + +""" Generate the full service URL including the resource path for a Garage S3 request. + +# Arguments +- `aws::GarageConfig` + The Garage configuration containing the endpoint. +- `service::String` + The AWS service name (e.g., `"s3"`). +- `resource::String` + The S3 resource path (e.g., `"/bucket/key"`). + +# Return +- `String`: The full URL combining the endpoint and resource path. +""" +function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String) + endpoint = strip(aws.endpoint, '/') + resource_path = startswith(resource, '/') ? resource : "/" * resource + return string(endpoint, resource_path) +end + +# =================================================================== +# 2. Thread-Safe Storage Client Wrapper +# =================================================================== + +""" Create a thread-safe Garage S3 storage client. + +# Arguments +- `endpoint::String` + The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`). +- `key::String` + The Garage access key ID (created via garage-ui). +- `secret::String` + The secret key corresponding to the access key. +- `bucket::String` + The name of the S3 bucket to use. + +# Keyword Arguments +- `region::String = "garage"` + The region identifier for the Garage instance. + +# Return +- `GarageStorage`: A thread-safe storage client instance. + +# Examples +```jldoctest +julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") +GarageStorage(GarageConfig(...), "sommpanion-s3") +``` +""" +struct GarageStorage + config::GarageConfig + bucket::String +end +function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage") + creds = SimpleCredentials(key, secret, "") + config = GarageConfig(endpoint, region, creds) + return GarageStorage(config, bucket) +end + + +# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl +function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}}) + return SHA.hmac_sha256(key, msg) +end + +function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}}) + return SHA.hmac_sha256(Vector{UInt8}(key), msg) +end + +# 2. AWSS3 overload targeting your custom Garage endpoint +function AWSS3._s3_sign_url_v4( + aws::GarageConfig, + bucket::String, + path::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + protocol::String="https", +) + path_escaped = URIs.escapepath("/$bucket/$path") + + now_datetime = now(Dates.UTC) + datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") + date_stamp = Dates.format(now_datetime, "YYYYmmdd") + + service = "s3" + scheme = "AWS4" + algorithm = "HMAC-SHA256" + terminator = "aws4_request" + + # Robust host & port extraction (prevents trailing colon bugs like 'domain.com:') + parsed_uri = URIs.URI(aws.endpoint) + has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0 + host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host + effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme + + scope = "$date_stamp/$(aws.region)/$service/$terminator" + + headers = OrderedDict{String,String}("Host" => host) + sort!(headers; by=name -> lowercase(name)) + canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") + + query = OrderedDict{String,String}( + "X-Amz-Expires" => string(seconds), + "X-Amz-Algorithm" => "$scheme-$algorithm", + "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope", + "X-Amz-Date" => datetime_stamp, + "X-Amz-SignedHeaders" => canonical_header_names, + ) + + if !isempty(aws.credentials.token) + query["X-Amz-Security-Token"] = aws.credentials.token + end + + sort!(query; by=name -> lowercase(name)) + + canonical_headers = join( + map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers)) + ) + + canonical_request = string( + "$verb\n", + "$path_escaped\n", + "$(URIs.escapeuri(query))\n", + "$canonical_headers\n", + "$canonical_header_names\n", + "UNSIGNED-PAYLOAD", + ) + + hashed_canonical_request = bytes2hex(sha256(canonical_request)) + + string_to_sign = string( + "$scheme-$algorithm\n", + "$datetime_stamp\n", + "$scope\n", + hashed_canonical_request, + ) + + # AWS Signature V4 Key Derivation Chain + key_secret = string(scheme, aws.credentials.secret_key) + key_date = _hmac_sha256(key_secret, date_stamp) + key_region = _hmac_sha256(key_date, aws.region) + key_service = _hmac_sha256(key_region, service) + key_signing = _hmac_sha256(key_service, terminator) + signature = _hmac_sha256(key_signing, string_to_sign) + + query["X-Amz-Signature"] = bytes2hex(signature) + + return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query)) +end + + + +""" + generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") + +Generates direct public HTTP GET URLs via Garage's s3-web interface. +Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint. +""" +function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") + clean_endpoint = strip(web_endpoint, '/') + clean_key = lstrip(key, '/') + + # s3-web URL format: https://// + download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)" + + return ( + download_url = download_url, + # DELETE requires s3-api (s3-web does not support HTTP DELETE) + delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE") + ) +end + + + + + +# 1. Initialize Garage configuration pointing to S3 API domain +storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key + "sommpanion-s3" +) + +# 2. Upload microservice file payload +put_file(storage, "job-9042-data.bin", payload_bytes) + +# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes) +urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900) + +# Pass `urls.download_url` and `urls.delete_url` to the downstream service \ No newline at end of file diff --git a/src/garageS3.jl b/src/garageS3.jl index b754213..cac33fd 100644 --- a/src/garageS3.jl +++ b/src/garageS3.jl @@ -21,7 +21,7 @@ storage = GeneralUtils.GarageStorage( ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) -GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}") +downloadUrl = GeneralUtils.put_file(storage, "users1.json", "{\"status\": \"active\"}") keys = GeneralUtils.list_files(storage) println("Read back: ", data) @@ -147,7 +147,7 @@ end The data to upload. # Return -- `String`: The file download URL if the upload succeeds. +- `NamedTuple{(:api, :web), Tuple{String, String}}`: The file download URLs if the upload succeeds (`api` for API access, `web` for web access). - `nothing` if the key contains slashes (upload aborted with warning). # Notes @@ -160,7 +160,7 @@ julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94 julia> put_file1 = put_file(storage1) julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json -"https://s3-api.yiem.cc/sommpanion-s3/test.json" +(api = "https://s3-api.yiem.cc/sommpanion-s3/test.json", web = "https://s3-web.yiem.cc/sommpanion-s3/test.json") ``` """ struct put_file @@ -177,10 +177,13 @@ function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}})::String AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) println("Successfully uploaded: ", key) - # file download URL - objURL = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key" + # file download URL using API + api = "$(storage.config.endpoint)/$(storage.bucket)/$key" - return objURL + # file download URL using browser + web = replace(api, "s3-api" => "s3-web") + + return (api=api, web=web) end """ Upload an object to the Garage S3 bucket. @@ -194,7 +197,7 @@ end The data to upload. # Return -- `String`: The file download URL if the upload succeeds. +- `NamedTuple{(:api, :web), Tuple{String, String}}`: The file download URLs if the upload succeeds (`api` for API access, `web` for web access). - `nothing` if the key contains slashes (upload aborted with warning). # Notes @@ -205,6 +208,7 @@ end ```julia julia> put_file(storage, "test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json +(api = "https://s3-api.yiem.cc/sommpanion-s3/test.json", web = "https://s3-web.yiem.cc/sommpanion-s3/test.json") ``` """ function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})::String @@ -218,10 +222,13 @@ function put_file(storage::GarageStorage, key::String, data::Union{String, Vecto AWSS3.s3_put(storage.config, storage.bucket, key, data) println("Successfully uploaded: ", key) - # file download URL - objURL = "$(storage.config.endpoint)/$(storage.bucket)/$key" + # file download URL using API + api = "$(storage.config.endpoint)/$(storage.bucket)/$key" - return objURL + # file download URL using browser + web = replace(api, "s3-api" => "s3-web") + + return (api=api, web=web) end """ Download an object from the Garage S3 bucket.