using SHA using Dates using URIs using OrderedCollections using AWS @service S3 using AWSS3 # Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key). # this file use local AWS config """ Example storage = GeneralUtils.GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key "sommpanion-s3" ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}") keys = GeneralUtils.list_files(storage) println("Read back: ", data) println("Bucket keys: ", keys) # test with curl curl -v \ -H 'Host: sommpanion-s3.s3-web.yiem.cc' \ http://192.168.88.106:3902/users-1002.json # test with API data = String(GeneralUtils.get_file(storage, "users-1005.json")) # test with a browser https://s3-web.mydomain.com/my_bucket_name/users-1005.json """ # =================================================================== # 1. Custom GarageConfig Definition (Thread-Safe, No Global State) # =================================================================== struct SimpleCredentials access_key_id::String secret_key::String token::String end struct GarageConfig <: AWS.AbstractAWSConfig endpoint::String region::String credentials::SimpleCredentials end # Required extensions for AWS.jl pipeline AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c AWS.credentials(c::SimpleCredentials) = c AWS.check_credentials(c::SimpleCredentials) = c AWS.region(aws::GarageConfig) = aws.region AWS.credentials(aws::GarageConfig) = aws.credentials """ Generate the base URL for a Garage S3 service request. # Arguments - `aws::GarageConfig` The Garage configuration containing the endpoint. - `service::String` The AWS service name (e.g., `"s3"`). - `region::String` The AWS region string. # Return - `String`: The stripped endpoint URL (trailing slashes removed). """ function AWS.generate_service_url(aws::GarageConfig, service::String, region::String) return strip(aws.endpoint, '/') end """ Generate the full service URL including the resource path for a Garage S3 request. # Arguments - `aws::GarageConfig` The Garage configuration containing the endpoint. - `service::String` The AWS service name (e.g., `"s3"`). - `resource::String` The S3 resource path (e.g., `"/bucket/key"`). # Return - `String`: The full URL combining the endpoint and resource path. """ function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String) endpoint = strip(aws.endpoint, '/') resource_path = startswith(resource, '/') ? resource : "/" * resource return string(endpoint, resource_path) end # =================================================================== # 2. Thread-Safe Storage Client Wrapper # =================================================================== """ Create a thread-safe Garage S3 storage client. # Arguments - `endpoint::String` The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`). - `key::String` The Garage access key ID (created via garage-ui). - `secret::String` The secret key corresponding to the access key. - `bucket::String` The name of the S3 bucket to use. # Keyword Arguments - `region::String = "garage"` The region identifier for the Garage instance. # Return - `GarageStorage`: A thread-safe storage client instance. # Examples ```jldoctest julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") GarageStorage(GarageConfig(...), "sommpanion-s3") ``` """ struct GarageStorage config::GarageConfig bucket::String end function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage") creds = SimpleCredentials(key, secret, "") config = GarageConfig(endpoint, region, creds) return GarageStorage(config, bucket) end # 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}}) return SHA.hmac_sha256(key, msg) end function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}}) return SHA.hmac_sha256(Vector{UInt8}(key), msg) end # 2. AWSS3 overload targeting your custom Garage endpoint function AWSS3._s3_sign_url_v4( aws::GarageConfig, bucket::String, path::String, seconds::Int=3600; verb::String="GET", content_type::String="application/octet-stream", protocol::String="https", ) path_escaped = URIs.escapepath("/$bucket/$path") now_datetime = now(Dates.UTC) datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") date_stamp = Dates.format(now_datetime, "YYYYmmdd") service = "s3" scheme = "AWS4" algorithm = "HMAC-SHA256" terminator = "aws4_request" # Robust host & port extraction (prevents trailing colon bugs like 'domain.com:') parsed_uri = URIs.URI(aws.endpoint) has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0 host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme scope = "$date_stamp/$(aws.region)/$service/$terminator" headers = OrderedDict{String,String}("Host" => host) sort!(headers; by=name -> lowercase(name)) canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") query = OrderedDict{String,String}( "X-Amz-Expires" => string(seconds), "X-Amz-Algorithm" => "$scheme-$algorithm", "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope", "X-Amz-Date" => datetime_stamp, "X-Amz-SignedHeaders" => canonical_header_names, ) if !isempty(aws.credentials.token) query["X-Amz-Security-Token"] = aws.credentials.token end sort!(query; by=name -> lowercase(name)) canonical_headers = join( map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers)) ) canonical_request = string( "$verb\n", "$path_escaped\n", "$(URIs.escapeuri(query))\n", "$canonical_headers\n", "$canonical_header_names\n", "UNSIGNED-PAYLOAD", ) hashed_canonical_request = bytes2hex(sha256(canonical_request)) string_to_sign = string( "$scheme-$algorithm\n", "$datetime_stamp\n", "$scope\n", hashed_canonical_request, ) # AWS Signature V4 Key Derivation Chain key_secret = string(scheme, aws.credentials.secret_key) key_date = _hmac_sha256(key_secret, date_stamp) key_region = _hmac_sha256(key_date, aws.region) key_service = _hmac_sha256(key_region, service) key_signing = _hmac_sha256(key_service, terminator) signature = _hmac_sha256(key_signing, string_to_sign) query["X-Amz-Signature"] = bytes2hex(signature) return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query)) end """ generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") Generates direct public HTTP GET URLs via Garage's s3-web interface. Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint. """ function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") clean_endpoint = strip(web_endpoint, '/') clean_key = lstrip(key, '/') # s3-web URL format: https://// download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)" return ( download_url = download_url, # DELETE requires s3-api (s3-web does not support HTTP DELETE) delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE") ) end # 1. Initialize Garage configuration pointing to S3 API domain storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key "sommpanion-s3" ) # 2. Upload microservice file payload put_file(storage, "job-9042-data.bin", payload_bytes) # 3. Generate pre-signed ephemeral URLs (valid for 15 minutes) urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900) # Pass `urls.download_url` and `urls.delete_url` to the downstream service