module garageS3 export LifecycleExpiration, build_lifecycle_xml, GarageStorage, s3PutFile, s3GetFile, s3ListFiles, s3DeleteFile, s3SetLifecycleExpiration, s3GetLifecycle, s3DeleteLifecycle, s3SignUrl, s3SignUrlWeb using AWS @service S3 using AWSS3 using Dates: Dates using URIs: URIs using OrderedCollections: OrderedDict using MbedTLS: digest, MD_SHA256 using Base64: bytes2hex # ---------------------------------------------- 100 --------------------------------------------- # # Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key). # this file use local AWS config """ Example storage1 = GeneralUtils.GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key "sommpanion" ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) # use UUID as filename because GarageS3 requires unique filename for each uploaded object urls = GeneralUtils.s3PutFile(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}") key = GeneralUtils.s3ListFiles(storage) println("Read back: ", data) println("Bucket keys: ", key) # test with curl curl -v \ -H 'Host: testbucket.s3-web.yiem.cc' \ http://192.168.88.106:3902/users-1002.json # test with API data = String(GeneralUtils.s3GetFile(storage, key[1])) # test with a browser (urls.web) https://s3-web.mydomain.com/my_bucket_name/users-1005.json """ # =================================================================== # 1. Custom GarageConfig Definition (Thread-Safe, No Global State) # =================================================================== mutable struct SimpleCredentials access_key_id::String secret_key::String token::String end mutable struct GarageConfig <: AWS.AbstractAWSConfig endpoint::String region::String credentials::SimpleCredentials end # Required extensions for AWS.jl pipeline AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c AWS.credentials(c::SimpleCredentials) = c AWS.check_credentials(c::SimpleCredentials) = c AWS.region(aws::GarageConfig) = aws.region AWS.credentials(aws::GarageConfig) = aws.credentials """ Generate the full service URL including the resource path for a Garage S3 request. # Arguments - `aws::GarageConfig` The Garage configuration containing the endpoint. - `service::String` The AWS service name (e.g., `"s3"`). - `resource::String` The S3 resource path (e.g., `"/bucket/key"`). # Return - `String`: The full URL combining the endpoint and resource path. """ function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String) endpoint = strip(aws.endpoint, '/') resource_path = startswith(resource, '/') ? resource : "/" * resource return string(endpoint, resource_path) end # =================================================================== # 2. Thread-Safe Storage Client Wrapper # =================================================================== """ Create a thread-safe Garage S3 storage client. # Arguments - `endpoint::String` The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`). - `key::String` The Garage access key ID (created via garage-ui). - `secret::String` The secret key corresponding to the access key. - `bucket::String` The name of the S3 bucket to use. # Keyword Arguments - `region::String = "garage"` The region identifier for the Garage instance. # Return - `GarageStorage`: A thread-safe storage client instance. # Examples ```jldoctest julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") GarageStorage(GarageConfig(...), "sommpanion-s3") ``` """ mutable struct GarageStorage config::GarageConfig bucket::String end function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage") creds = SimpleCredentials(key, secret, "") config = GarageConfig(endpoint, region, creds) return GarageStorage(config, bucket) end """ Upload an object to the Garage S3 bucket (callable struct). # Arguments - `key::String` The object key (name) in the bucket. - `data::Union{String, Vector{UInt8}}` The data to upload. # Return - `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs. # Examples ```jldoctest julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") julia> s3PutFile1 = s3PutFile(storage1) julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> downloadUrl.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" julia> downloadUrl.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ mutable struct s3PutFile storage::GarageStorage end function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) println("Successfully uploaded: ", key) # object url api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key" # file download URL using browser web = replace(api, "s3-api" => "s3-web") return (api=api, web=web) end """ Upload an object to the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. - `key::String` The object key (name) in the bucket. - `data::Union{String, Vector{UInt8}}` The data to upload. # Return - `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs. # Examples ```julia julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> result.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" julia> result.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(storage.config, storage.bucket, key, data) println("Successfully uploaded: ", key) # object url api = "$(storage.config.endpoint)/$(storage.bucket)/$key" # file download URL using browser web = replace(api, "s3-api" => "s3-web") return (api=api, web=web) end """ Download an object from the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. - `key::String` The object key (name) in the bucket. # Return - `Vector{UInt8}`: The raw bytes of the downloaded object. # Examples ```julia julia> data = String(s3GetFile(storage, "test.json")) "\"{\\\"status\\\": \\\"active\\\"}\"" ``` """ function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} try return AWSS3.s3_get(storage.config, storage.bucket, key) catch return nothing end end """ List all object keys in the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. # Return - `Vector{String}`: A list of object keys (file names) in the bucket. # Examples ```julia julia> keys = s3ListFiles(storage) 3-element Vector{String}: "test1.json" "test2.json" "users-1002.json" ``` """ function s3ListFiles(storage::GarageStorage) # Use delimiter="" to get all individual keys (no grouping by "directory") objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="") return [obj["Key"] for obj in objects] end """ Delete an object from the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. - `key::String` The object key (name) to delete. # Examples ```julia julia> s3DeleteFile(storage, "test.json") ``` """ function s3DeleteFile(storage::GarageStorage, key::String) AWSS3.s3_delete(storage.config, storage.bucket, key) end """ s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String Generate a pre-signed URL for an object in the Garage S3 bucket. A pre-signed URL grants temporary access without AWS credentials. Signature is computed using AWS Signature Version 4 (HMAC-SHA256). # Arguments - `storage::GarageStorage`: Storage client from `GarageStorage(endpoint, key, secret, bucket)`. - `key::String`: Object key. Supports nested paths like `"folder/subfolder/file.txt"`. - `seconds::Int`: URL validity in seconds. Default: 3600. # Keyword Arguments - `verb::String = "GET"`: HTTP method (`"GET"` downloads, `"PUT"` uploads). - `content_type::String = "application/octet-stream"`: Required `Content-Type` header. For PUT, set to the file's MIME type (`"image/png"`, `"application/json"`, etc.). - `signature_version::String = "v4"`: Always use `"v4"` (v2 is deprecated). # Return `String`: The pre-signed URL. # Examples ```julia using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "sommpanion" ) # Upload — PUT with curl url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg") # curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' # Download — GET with curl url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET") # curl → returns file content # Upload JSON url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json") # curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' # Delete url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE") # curl -X DELETE url # Multiple uploads (chat app) urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300; verb="PUT", content_type="image/png") for _ in 1:10] # Signature differs by verb and content_type url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET") url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain") ``` # Notes - URLs are valid until `seconds` expires; they can be used **multiple times** (not one-time). - Server clocks must be within ~5 minutes of the Garage server (UTC time in signature). - PUT uploads require the client's `Content-Type` to match `content_type`. - For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`. """ function s3SignUrl( storage::GarageStorage, key::String, seconds::Int=3600; verb::String="GET", content_type::String="application/octet-stream", signature_version::String="v4", )::String config = storage.config creds = config.credentials bucket = storage.bucket region = config.region endpoint = strip(config.endpoint, '/') scheme = "AWS4" algorithm = "HMAC-SHA256" terminator = "aws4_request" service = "s3" now_datetime = Dates.now(Dates.UTC) datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") date_stamp = Dates.format(now_datetime, "YYYYmmdd") scope = "$date_stamp/$region/$service/$terminator" headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host) if verb == "PUT" headers["Content-Type"] = content_type end sort!(headers; by=name -> lowercase(name)) canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") query = OrderedDict{String,String}( "X-Amz-Expires" => string(seconds), "X-Amz-Algorithm" => "$scheme-$algorithm", "X-Amz-Credential" => "$(creds.access_key_id)/$scope", "X-Amz-Date" => datetime_stamp, "X-Amz-SignedHeaders" => canonical_header_names, ) if !isempty(creds.token) query["X-Amz-Security-Token"] = creds.token end canonical_headers = join( map( header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers), ), ) resource_path = "/$bucket/$key" # Build canonical query string: sort by key=value pair lexicographically, # URL-encoding each key and value (matching Garage's uri_encode) sorted_q = sort( [URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)], by=lowercase, ) canonical_query = join(sorted_q, "&") canonical_request = string( "$verb\n", resource_path, "\n", canonical_query, "\n", canonical_headers, "\n", canonical_header_names, "\n", "UNSIGNED-PAYLOAD", ) string_to_sign = string( "$scheme-$algorithm\n", "$datetime_stamp\n", "$scope\n", bytes2hex(digest(MD_SHA256, canonical_request)), ) key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key)) key_date = digest(MD_SHA256, date_stamp, key_secret_bytes) key_region = digest(MD_SHA256, region, key_date) key_service = digest(MD_SHA256, service, key_region) key_signing = digest(MD_SHA256, terminator, key_service) signature = digest(MD_SHA256, string_to_sign, key_signing) query["X-Amz-Signature"] = bytes2hex(signature) query_string = URIs.escapeuri(query) return "$(endpoint)/$bucket/$key?$query_string" end """ Generate a pre-signed URL using the Garage S3 web endpoint (s3-web). Useful for direct browser access — URLs look like `https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...` Arguments and keyword arguments are the same as `s3SignUrl`. """ function s3SignUrlWeb( storage::GarageStorage, key::String, seconds::Int=3600; verb::String="GET", content_type::String="application/octet-stream", signature_version::String="v4", )::String web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web") web_creds = SimpleCredentials( storage.config.credentials.access_key_id, storage.config.credentials.secret_key, storage.config.credentials.token, ) web_config = GarageConfig(web_endpoint, storage.config.region, web_creds) web_storage = GarageStorage(web_config, storage.bucket) return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version) end # =================================================================== # 3. Bucket Lifecycle Configuration # =================================================================== """ Lifecycle expiration configuration for a bucket. # Fields - `prefix::String`: Object key prefix this rule applies to. - `days::Int`: Number of days after object creation to expire/delete. - `enabled::Bool`: Whether the rule is active (default: `true`). - `id::String`: Unique identifier for the rule (default: auto-generated). """ mutable struct LifecycleExpiration prefix::String days::Int enabled::Bool id::String end function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="") rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id return LifecycleExpiration(prefix, days, enabled, rule_id) end """ Build the XML body for a PutBucketLifecycleConfiguration request. Supports a single rule with expiration by prefix. """ function build_lifecycle_xml(rule::LifecycleExpiration)::String status = rule.enabled ? "Enabled" : "Disabled" return """ $(rule.id) $(rule.prefix) $status $(rule.days) """ end """ Set bucket lifecycle expiration rule. # Arguments - `storage::GarageStorage`: The storage client instance. - `rule::LifecycleExpiration`: The lifecycle expiration rule. # Returns - `Dict{String, Any}`: Parsed response from the server. # Example ```julia rule = LifecycleExpiration("uploads/", 30) s3SetLifecycleExpiration(storage, rule) ``` """ function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration) xml_body = build_lifecycle_xml(rule) args = Dict{String,Any}("body" => xml_body) return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config) end """ Get the bucket lifecycle configuration. # Arguments - `storage::GarageStorage`: The storage client instance. # Returns - `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured. # Example ```julia config = s3GetLifecycle(storage) ``` """ function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} try return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config) catch e if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration" return nothing end rethrow(e) end end """ Delete the bucket lifecycle configuration. # Arguments - `storage::GarageStorage`: The storage client instance. # Returns - `Dict{String, Any}`: Parsed response from the server. # Example ```julia s3DeleteLifecycle(storage) ``` """ function s3DeleteLifecycle(storage::GarageStorage) return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config) end end # module GarageS3