module garageS3 export LifecycleExpiration, build_lifecycle_xml, GarageStorage, put_file, get_file, list_files, delete_file, set_lifecycle_expiration, get_lifecycle, delete_lifecycle, s3_sign_url using AWS @service S3 using AWSS3 using Dates: Dates using URIs: URIs using OrderedCollections: OrderedDict using MbedTLS: digest, MD_SHA256 using Base64: bytes2hex # ---------------------------------------------- 100 --------------------------------------------- # # Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key). # this file use local AWS config """ Example storage1 = GeneralUtils.GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key "testbucket" ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) # use UUID as filename because GarageS3 requires unique filename for each uploaded object urls = GeneralUtils.put_file(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}") key = GeneralUtils.list_files(storage) println("Read back: ", data) println("Bucket keys: ", key) # test with curl curl -v \ -H 'Host: testbucket.s3-web.yiem.cc' \ http://192.168.88.106:3902/users-1002.json # test with API data = String(GeneralUtils.get_file(storage, key[1])) # test with a browser (urls.web) https://s3-web.mydomain.com/my_bucket_name/users-1005.json """ # =================================================================== # 1. Custom GarageConfig Definition (Thread-Safe, No Global State) # =================================================================== mutable struct SimpleCredentials access_key_id::String secret_key::String token::String end mutable struct GarageConfig <: AWS.AbstractAWSConfig endpoint::String region::String credentials::SimpleCredentials end # Required extensions for AWS.jl pipeline AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c AWS.credentials(c::SimpleCredentials) = c AWS.check_credentials(c::SimpleCredentials) = c AWS.region(aws::GarageConfig) = aws.region AWS.credentials(aws::GarageConfig) = aws.credentials """ Generate the full service URL including the resource path for a Garage S3 request. # Arguments - `aws::GarageConfig` The Garage configuration containing the endpoint. - `service::String` The AWS service name (e.g., `"s3"`). - `resource::String` The S3 resource path (e.g., `"/bucket/key"`). # Return - `String`: The full URL combining the endpoint and resource path. """ function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String) endpoint = strip(aws.endpoint, '/') resource_path = startswith(resource, '/') ? resource : "/" * resource return string(endpoint, resource_path) end # =================================================================== # 2. Thread-Safe Storage Client Wrapper # =================================================================== """ Create a thread-safe Garage S3 storage client. # Arguments - `endpoint::String` The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`). - `key::String` The Garage access key ID (created via garage-ui). - `secret::String` The secret key corresponding to the access key. - `bucket::String` The name of the S3 bucket to use. # Keyword Arguments - `region::String = "garage"` The region identifier for the Garage instance. # Return - `GarageStorage`: A thread-safe storage client instance. # Examples ```jldoctest julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") GarageStorage(GarageConfig(...), "sommpanion-s3") ``` """ mutable struct GarageStorage config::GarageConfig bucket::String end function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage") creds = SimpleCredentials(key, secret, "") config = GarageConfig(endpoint, region, creds) return GarageStorage(config, bucket) end """ Upload an object to the Garage S3 bucket (callable struct). # Arguments - `key::String` The object key (name) in the bucket. - `data::Union{String, Vector{UInt8}}` The data to upload. # Return - `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs. # Examples ```jldoctest julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") julia> put_file1 = put_file(storage1) julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> downloadUrl.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" julia> downloadUrl.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ mutable struct put_file storage::GarageStorage end function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) println("Successfully uploaded: ", key) # object url api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key" # file download URL using browser web = replace(api, "s3-api" => "s3-web") return (api=api, web=web) end """ Upload an object to the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. - `key::String` The object key (name) in the bucket. - `data::Union{String, Vector{UInt8}}` The data to upload. # Return - `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs. # Examples ```julia julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> result.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" julia> result.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(storage.config, storage.bucket, key, data) println("Successfully uploaded: ", key) # object url api = "$(storage.config.endpoint)/$(storage.bucket)/$key" # file download URL using browser web = replace(api, "s3-api" => "s3-web") return (api=api, web=web) end """ Download an object from the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. - `key::String` The object key (name) in the bucket. # Return - `Vector{UInt8}`: The raw bytes of the downloaded object. # Examples ```julia julia> data = String(get_file(storage, "test.json")) "\"{\\\"status\\\": \\\"active\\\"}\"" ``` """ function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} try return AWSS3.s3_get(storage.config, storage.bucket, key) catch return nothing end end """ List all object keys in the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. # Return - `Vector{String}`: A list of object keys (file names) in the bucket. # Examples ```julia julia> keys = list_files(storage) 3-element Vector{String}: "test1.json" "test2.json" "users-1002.json" ``` """ function list_files(storage::GarageStorage) # Use delimiter="" to get all individual keys (no grouping by "directory") objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="") return [obj["Key"] for obj in objects] end """ Delete an object from the Garage S3 bucket. # Arguments - `storage::GarageStorage` The storage client instance. - `key::String` The object key (name) to delete. # Examples ```julia julia> delete_file(storage, "test.json") ``` """ function delete_file(storage::GarageStorage, key::String) AWSS3.s3_delete(storage.config, storage.bucket, key) end """ s3_sign_url(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String Generate a pre-signed URL for an object in the Garage S3 bucket. A pre-signed URL is a signed URL that grants temporary access to an S3 object without requiring AWS credentials. The URL expires after the specified number of seconds. The signature is computed using AWS Signature Version 4 (HMAC-SHA256). # Why use pre-signed URLs? Pre-signed URLs let you grant temporary, scoped access to objects without sharing your API credentials. Typical use cases: - **Direct browser downloads**: Return the URL to a frontend so users can download files without going through your server. - **Direct browser uploads**: Generate a PUT pre-signed URL, pass it to the frontend, and let the user upload a file directly to S3 with a `curl` or `fetch` call. - **Temporary sharing**: Send an expiring link to someone who should not have permanent access to the object. - **Server-side download without round-trip**: Instead of downloading the file to your server and re-uploading it elsewhere, pass the pre-signed URL to another service. # How it works The function computes an HMAC-SHA256 signature over a canonical request that includes the HTTP verb, the object path, the query parameters, and the host header. The signature is appended to the URL as `X-Amz-Signature`. Garage (and any S3 compatible service) verifies the signature before serving the object. # URL format https://s3-api.your-garage-instance.com/bucket/key?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...&X-Amz-Date=...&X-Amz-Expires=...&X-Amz-SignedHeaders=...&X-Amz-Signature=... # Arguments - `storage::GarageStorage` A storage client created with `GarageStorage(endpoint, key, secret, bucket)`. - `key::String` The object key (file name) in the bucket. Supports nested paths like `"folder/subfolder/file.txt"`. - `seconds::Int` How long the URL remains valid, in seconds. Default: 3600 (1 hour). # Keyword Arguments - `verb::String = "GET"` The HTTP method the URL permits. Use `"GET"` for downloads (the default) or `"PUT"` for uploads. Other values like `"DELETE"` are also supported. - `content_type::String = "application/octet-stream"` The `Content-Type` header value that must match when the URL is used. For PUT uploads, set this to the actual MIME type of the file you plan to upload (e.g. `"text/plain"`, `"application/json"`, `"image/png"`). - `signature_version::String = "v4"` Signature algorithm version. Always use `"v4"` — version 2 is deprecated and not supported by many S3-compatible services. # Return - `String`: The full pre-signed URL, ready to use with `curl`, a browser, or any HTTP client. # Examples ### Download link (GET — default) ```julia storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket") # 1-hour download link url = s3_sign_url(storage, "documents/report.pdf", 3600) println("Download link: ", url) # Use in a browser — opens the file directly # > open(url) # Use with curl from the terminal # > curl -L "url" -o report.pdf ``` ### Upload link (PUT) ```julia # Generate a PUT URL valid for 10 minutes, accepting text/plain uploads url = s3_sign_url(storage, "uploads/user-photo.png", 600; verb="PUT", content_type="image/png") # The user (or your frontend) uploads a file like this: # > curl -X PUT -H "Content-Type: image/png" --data @photo.png "url" ``` ### Embedding in HTML ```julia url = s3_sign_url(storage, "shared/shared-file.zip", 86400) # 24 hours # Use the URL in HTML: Download shared-file.zip (valid 24 hours) ``` ### Verification — signature differs by verb and content-type ```julia url_get = s3_sign_url(storage, "file.txt", 3600; verb="GET") url_put = s3_sign_url(storage, "file.txt", 3600; verb="PUT", content_type="text/plain") # url_get and url_put have different X-Amz-Signature values because the # canonical request (and thus the HMAC) is different. ``` # Notes - The pre-signed URL is **not encrypted** — anyone with the URL can access the object until it expires. Treat the URL like a temporary password. - The signature includes the current UTC time (`X-Amz-Date`), so both the generating machine and the Garage server should have reasonably synchronized clocks (± a few minutes tolerance). - Only `v4` signature is implemented. Garage supports v4 natively. - For cross-origin (CORS) uploads from a browser, you also need the Garage bucket to have CORS rules that allow the `Origin` and `Method` headers. # s3_sign_url vs get_file — when to use which | Scenario | Use | |---|---| | Your server needs to read/write an object | `get_file(storage, key)`, `put_file(storage, key, data)` | | A browser or external client needs to download an object | `s3_sign_url(storage, key, 3600; verb="GET")` | | A browser or external client needs to upload a file | `s3_sign_url(storage, key, 600; verb="PUT", content_type="image/png")` | | You need to pass a download link to a third party | `s3_sign_url(storage, key, 86400)` — no credentials shared | `get_file` / `put_file` use your API credentials on every call. `s3_sign_url` embeds a time-limited signature in the URL — no credentials needed. # Complete upload workflow example ```julia # 1. Generate a PUT pre-signed URL on your server url = s3_sign_url(storage, "avatars/user-12345.png", 300; verb="PUT", content_type="image/png") # 2. Send `url` to your frontend (JSON response, React state, etc.) # 3. Frontend uploads directly to Garage: # fetch(url, { method: "PUT", headers: {"Content-Type": "image/png"}, body: fileBlob }) # or: # curl -X PUT -H "Content-Type: image/png" --data @photo.png url # 4. Optional — verify the upload succeeded # data = get_file(storage, "avatars/user-12345.png") ``` # Complete download workflow example ```julia # 1. Generate a GET pre-signed URL url = s3_sign_url(storage, "reports/quarterly.pdf", 7200) # 2 hours # 2a. Redirect user to the URL (S3 returns 302) # return redirect(url) # 2b. Embed in HTML link # Download quarterly.pdf # 2c. Use with curl # curl -L url -o quarterly.pdf ``` # Query parameters in the generated URL | Parameter | Purpose | |---|---| | `X-Amz-Algorithm` | Must be `AWS4-HMAC-SHA256` | | `X-Amz-Credential` | Your access key ID + date + region + service + request token | | `X-Amz-Date` | Timestamp when the URL was generated (UTC, ISO 8601 format) | | `X-Amz-Expires` | Number of seconds until expiration | | `X-Amz-Security-Token` | Optional session token (empty if not provided) | | `X-Amz-SignedHeaders` | Semicolon-separated list of signed headers (currently only `host`) | | `X-Amz-Signature` | The HMAC-SHA256 signature — the core authorization token | # Common pitfalls - **Clock skew**: If the server and Garage are more than ~5 minutes out of sync, the signature will be rejected. - **Content-Type mismatch**: When using a PUT URL, the `Content-Type` sent by the client must match the `content_type` you passed to `s3_sign_url`. - **URL encoding**: The key is URL-encoded in the path. Keys with spaces or special characters work correctly, but do not double-encode the returned URL. - **Single use**: Pre-signed URLs can be used multiple times until expiration; they are not one-time tokens. """ function s3_sign_url( storage::GarageStorage, key::String, seconds::Int=3600; verb::String="GET", content_type::String="application/octet-stream", signature_version::String="v4", )::String config = storage.config creds = config.credentials bucket = storage.bucket region = config.region endpoint = strip(config.endpoint, '/') scheme = "AWS4" algorithm = "HMAC-SHA256" terminator = "aws4_request" service = "s3" now_datetime = Dates.now(Dates.UTC) datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") date_stamp = Dates.format(now_datetime, "YYYYmmdd") scope = "$date_stamp/$region/$service/$terminator" headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host) sort!(headers; by=name -> lowercase(name)) canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") query = OrderedDict{String,String}( "X-Amz-Expires" => string(seconds), "X-Amz-Algorithm" => "$scheme-$algorithm", "X-Amz-Credential" => "$(creds.access_key_id)/$scope", "X-Amz-Date" => datetime_stamp, "X-Amz-Security-Token" => isempty(creds.token) ? "" : creds.token, "X-Amz-SignedHeaders" => canonical_header_names, ) sort!(query; by=name -> lowercase(name)) canonical_headers = join( map( header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers), ), ) resource_path = "/$bucket/$key" canonical_request = string( "$verb\n", "$(URIs.escapeuri(resource_path))\n", "$(URIs.escapeuri(query))\n", "$canonical_headers\n", "$canonical_header_names\n", "UNSIGNED-PAYLOAD", ) string_to_sign = string( "$scheme-$algorithm\n", "$datetime_stamp\n", "$scope\n", bytes2hex(digest(MD_SHA256, canonical_request)), ) key_secret = string(scheme, creds.secret_key) key_date = digest(MD_SHA256, date_stamp, key_secret) key_region = digest(MD_SHA256, region, key_date) key_service = digest(MD_SHA256, service, key_region) key_signing = digest(MD_SHA256, terminator, key_service) signature = digest(MD_SHA256, string_to_sign, key_signing) query["X-Amz-Signature"] = bytes2hex(signature) query_string = URIs.escapeuri(query) return "$(endpoint)/$bucket/$key?$query_string" end # =================================================================== # 3. Bucket Lifecycle Configuration # =================================================================== """ Lifecycle expiration configuration for a bucket. # Fields - `prefix::String`: Object key prefix this rule applies to. - `days::Int`: Number of days after object creation to expire/delete. - `enabled::Bool`: Whether the rule is active (default: `true`). - `id::String`: Unique identifier for the rule (default: auto-generated). """ mutable struct LifecycleExpiration prefix::String days::Int enabled::Bool id::String end function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="") rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id return LifecycleExpiration(prefix, days, enabled, rule_id) end """ Build the XML body for a PutBucketLifecycleConfiguration request. Supports a single rule with expiration by prefix. """ function build_lifecycle_xml(rule::LifecycleExpiration)::String status = rule.enabled ? "Enabled" : "Disabled" return """ $(rule.id) $(rule.prefix) $status $(rule.days) """ end """ Set bucket lifecycle expiration rule. # Arguments - `storage::GarageStorage`: The storage client instance. - `rule::LifecycleExpiration`: The lifecycle expiration rule. # Returns - `Dict{String, Any}`: Parsed response from the server. # Example ```julia rule = LifecycleExpiration("uploads/", 30) set_lifecycle_expiration(storage, rule) ``` """ function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration) xml_body = build_lifecycle_xml(rule) args = Dict{String,Any}("body" => xml_body) return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config) end """ Get the bucket lifecycle configuration. # Arguments - `storage::GarageStorage`: The storage client instance. # Returns - `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured. # Example ```julia config = get_lifecycle(storage) ``` """ function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} try return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config) catch e if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration" return nothing end rethrow(e) end end """ Delete the bucket lifecycle configuration. # Arguments - `storage::GarageStorage`: The storage client instance. # Returns - `Dict{String, Any}`: Parsed response from the server. # Example ```julia delete_lifecycle(storage) ``` """ function delete_lifecycle(storage::GarageStorage) return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config) end end # module GarageS3