diff --git a/Manifest.toml b/Manifest.toml index 7ff0445..0d17d24 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.7" manifest_format = "2.0" -project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044" +project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2" [[deps.AWS]] deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"] diff --git a/Project.toml b/Project.toml index f074c5a..0b22b6a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,11 +1,12 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" version = "0.6.9" -authors = ["tonaerospace "] +authors = ["tonaerospace String + +Generate a pre-signed URL for an object in the Garage S3 bucket. + +A pre-signed URL is a signed URL that grants temporary access to an S3 object without +requiring AWS credentials. The URL expires after the specified number of seconds. +The signature is computed using AWS Signature Version 4 (HMAC-SHA256). + +# Why use pre-signed URLs? + +Pre-signed URLs let you grant temporary, scoped access to objects without sharing +your API credentials. Typical use cases: + +- **Direct browser downloads**: Return the URL to a frontend so users can download + files without going through your server. +- **Direct browser uploads**: Generate a PUT pre-signed URL, pass it to the frontend, + and let the user upload a file directly to S3 with a `curl` or `fetch` call. +- **Temporary sharing**: Send an expiring link to someone who should not have + permanent access to the object. +- **Server-side download without round-trip**: Instead of downloading the file to + your server and re-uploading it elsewhere, pass the pre-signed URL to another + service. + +# How it works + +The function computes an HMAC-SHA256 signature over a canonical request that +includes the HTTP verb, the object path, the query parameters, and the host header. +The signature is appended to the URL as `X-Amz-Signature`. Garage (and any S3 +compatible service) verifies the signature before serving the object. + +# URL format +https://s3-api.your-garage-instance.com/bucket/key?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...&X-Amz-Date=...&X-Amz-Expires=...&X-Amz-SignedHeaders=...&X-Amz-Signature=... + +# Arguments +- `storage::GarageStorage` + A storage client created with `GarageStorage(endpoint, key, secret, bucket)`. +- `key::String` + The object key (file name) in the bucket. Supports nested paths like + `"folder/subfolder/file.txt"`. +- `seconds::Int` + How long the URL remains valid, in seconds. Default: 3600 (1 hour). + +# Keyword Arguments +- `verb::String = "GET"` + The HTTP method the URL permits. Use `"GET"` for downloads (the default) or + `"PUT"` for uploads. Other values like `"DELETE"` are also supported. +- `content_type::String = "application/octet-stream"` + The `Content-Type` header value that must match when the URL is used. + For PUT uploads, set this to the actual MIME type of the file you plan to + upload (e.g. `"text/plain"`, `"application/json"`, `"image/png"`). +- `signature_version::String = "v4"` + Signature algorithm version. Always use `"v4"` — version 2 is deprecated + and not supported by many S3-compatible services. + +# Return +- `String`: The full pre-signed URL, ready to use with `curl`, a browser, or any + HTTP client. + +# Examples + +### Download link (GET — default) +```julia +storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket") + +# 1-hour download link +url = s3_sign_url(storage, "documents/report.pdf", 3600) +println("Download link: ", url) + +# Use in a browser — opens the file directly +# > open(url) + +# Use with curl from the terminal +# > curl -L "url" -o report.pdf +``` + +### Upload link (PUT) +```julia +# Generate a PUT URL valid for 10 minutes, accepting text/plain uploads +url = s3_sign_url(storage, "uploads/user-photo.png", 600; + verb="PUT", content_type="image/png") + +# The user (or your frontend) uploads a file like this: +# > curl -X PUT -H "Content-Type: image/png" --data @photo.png "url" +``` + +### Embedding in HTML +```julia +url = s3_sign_url(storage, "shared/shared-file.zip", 86400) # 24 hours +# Use the URL in HTML: Download shared-file.zip (valid 24 hours) +``` + +### Verification — signature differs by verb and content-type +```julia +url_get = s3_sign_url(storage, "file.txt", 3600; verb="GET") +url_put = s3_sign_url(storage, "file.txt", 3600; verb="PUT", content_type="text/plain") +# url_get and url_put have different X-Amz-Signature values because the +# canonical request (and thus the HMAC) is different. +``` + +# Notes +- The pre-signed URL is **not encrypted** — anyone with the URL can access the + object until it expires. Treat the URL like a temporary password. +- The signature includes the current UTC time (`X-Amz-Date`), so both the + generating machine and the Garage server should have reasonably synchronized + clocks (± a few minutes tolerance). +- Only `v4` signature is implemented. Garage supports v4 natively. +- For cross-origin (CORS) uploads from a browser, you also need the Garage bucket + to have CORS rules that allow the `Origin` and `Method` headers. + +# s3_sign_url vs get_file — when to use which + +| Scenario | Use | +|---|---| +| Your server needs to read/write an object | `get_file(storage, key)`, `put_file(storage, key, data)` | +| A browser or external client needs to download an object | `s3_sign_url(storage, key, 3600; verb="GET")` | +| A browser or external client needs to upload a file | `s3_sign_url(storage, key, 600; verb="PUT", content_type="image/png")` | +| You need to pass a download link to a third party | `s3_sign_url(storage, key, 86400)` — no credentials shared | + +`get_file` / `put_file` use your API credentials on every call. +`s3_sign_url` embeds a time-limited signature in the URL — no credentials needed. + +# Complete upload workflow example + +```julia +# 1. Generate a PUT pre-signed URL on your server +url = s3_sign_url(storage, "avatars/user-12345.png", 300; + verb="PUT", content_type="image/png") + +# 2. Send `url` to your frontend (JSON response, React state, etc.) + +# 3. Frontend uploads directly to Garage: +# fetch(url, { method: "PUT", headers: {"Content-Type": "image/png"}, body: fileBlob }) +# or: +# curl -X PUT -H "Content-Type: image/png" --data @photo.png url + +# 4. Optional — verify the upload succeeded +# data = get_file(storage, "avatars/user-12345.png") +``` + +# Complete download workflow example + +```julia +# 1. Generate a GET pre-signed URL +url = s3_sign_url(storage, "reports/quarterly.pdf", 7200) # 2 hours + +# 2a. Redirect user to the URL (S3 returns 302) +# return redirect(url) + +# 2b. Embed in HTML link +# Download quarterly.pdf + +# 2c. Use with curl +# curl -L url -o quarterly.pdf +``` + +# Query parameters in the generated URL + +| Parameter | Purpose | +|---|---| +| `X-Amz-Algorithm` | Must be `AWS4-HMAC-SHA256` | +| `X-Amz-Credential` | Your access key ID + date + region + service + request token | +| `X-Amz-Date` | Timestamp when the URL was generated (UTC, ISO 8601 format) | +| `X-Amz-Expires` | Number of seconds until expiration | +| `X-Amz-Security-Token` | Optional session token (empty if not provided) | +| `X-Amz-SignedHeaders` | Semicolon-separated list of signed headers (currently only `host`) | +| `X-Amz-Signature` | The HMAC-SHA256 signature — the core authorization token | + +# Common pitfalls + +- **Clock skew**: If the server and Garage are more than ~5 minutes out of sync, + the signature will be rejected. +- **Content-Type mismatch**: When using a PUT URL, the `Content-Type` sent by the + client must match the `content_type` you passed to `s3_sign_url`. +- **URL encoding**: The key is URL-encoded in the path. Keys with spaces or special + characters work correctly, but do not double-encode the returned URL. +- **Single use**: Pre-signed URLs can be used multiple times until expiration; + they are not one-time tokens. +""" +function s3_sign_url( + storage::GarageStorage, + key::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + signature_version::String="v4", +)::String + config = storage.config + creds = config.credentials + bucket = storage.bucket + region = config.region + endpoint = strip(config.endpoint, '/') + scheme = "AWS4" + algorithm = "HMAC-SHA256" + terminator = "aws4_request" + service = "s3" + + now_datetime = Dates.now(Dates.UTC) + datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") + date_stamp = Dates.format(now_datetime, "YYYYmmdd") + + scope = "$date_stamp/$region/$service/$terminator" + + headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host) + sort!(headers; by=name -> lowercase(name)) + canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") + + query = OrderedDict{String,String}( + "X-Amz-Expires" => string(seconds), + "X-Amz-Algorithm" => "$scheme-$algorithm", + "X-Amz-Credential" => "$(creds.access_key_id)/$scope", + "X-Amz-Date" => datetime_stamp, + "X-Amz-Security-Token" => isempty(creds.token) ? "" : creds.token, + "X-Amz-SignedHeaders" => canonical_header_names, + ) + + sort!(query; by=name -> lowercase(name)) + + canonical_headers = join( + map( + header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", + collect(headers), + ), + ) + + resource_path = "/$bucket/$key" + + canonical_request = string( + "$verb\n", + "$(URIs.escapeuri(resource_path))\n", + "$(URIs.escapeuri(query))\n", + "$canonical_headers\n", + "$canonical_header_names\n", + "UNSIGNED-PAYLOAD", + ) + + string_to_sign = string( + "$scheme-$algorithm\n", + "$datetime_stamp\n", + "$scope\n", + bytes2hex(digest(MD_SHA256, canonical_request)), + ) + + key_secret = string(scheme, creds.secret_key) + key_date = digest(MD_SHA256, date_stamp, key_secret) + key_region = digest(MD_SHA256, region, key_date) + key_service = digest(MD_SHA256, service, key_region) + key_signing = digest(MD_SHA256, terminator, key_service) + signature = digest(MD_SHA256, string_to_sign, key_signing) + + query["X-Amz-Signature"] = bytes2hex(signature) + + query_string = URIs.escapeuri(query) + return "$(endpoint)/$bucket/$key?$query_string" +end + # =================================================================== # 3. Bucket Lifecycle Configuration diff --git a/test/runtests.jl b/test/runtests.jl index 8ef2cf7..6f0c92f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using GeneralUtils.garageS3 using AWS using Random: randstring using UUIDs: uuid4 +using URIs @testset "garageS3.jl" begin @@ -460,4 +461,129 @@ using UUIDs: uuid4 delete_lifecycle(storage) end + # --- Pre-signed URL tests --- + + @testset "s3_sign_url returns a string" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "test.json", 3600) + @test url isa String + @test !isempty(url) + end + + @testset "s3_sign_url URL format contains correct endpoint, bucket, key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "mykey.json", 1800) + @test occursin("https://s3-api.yiem.cc", url) + @test occursin("/testbucket/mykey.json?", url) + end + + @testset "s3_sign_url contains required query parameters" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "test.json", 3600) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url) + @test occursin("X-Amz-Expires=3600", url) + @test occursin("X-Amz-Credential=", url) + @test occursin("X-Amz-Date=", url) + @test occursin("X-Amz-SignedHeaders=", url) + @test occursin("X-Amz-Signature=", url) + end + + @testset "s3_sign_url with custom expiry duration" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "test.json", 600) + @test occursin("X-Amz-Expires=600", url) + end + + @testset "s3_sign_url with GET verb produces different signature than default" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url_get = s3_sign_url(storage, "test.json", 3600; verb="GET") + url_default = s3_sign_url(storage, "test.json", 3600) + # GET verb should produce a valid signature + @test occursin("X-Amz-Signature=", url_get) + @test occursin("X-Amz-Signature=", url_default) + # Both URLs should have the same structure but potentially different signatures + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get) + end + + @testset "s3_sign_url with PUT verb and content_type" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url_put = s3_sign_url(storage, "upload.json", 300; verb="PUT", content_type="application/json") + url_get = s3_sign_url(storage, "upload.json", 300; verb="GET") + # PUT and GET should produce different signatures (different canonical request) + @test occursin("X-Amz-Signature=", url_put) + @test occursin("X-Amz-Signature=", url_get) + end + + @testset "s3_sign_url with special characters in key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + test_key = "folder/sub folder/file with spaces.json" + url = s3_sign_url(storage, test_key, 3600) + @test occursin("testbucket/", url) + end + + @testset "s3_sign_url with deeply nested key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + test_key = "a/b/c/d/e/deep-file.json" + url = s3_sign_url(storage, test_key, 3600) + @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url) + end + + @testset "integration: pre-signed URL can download existing file" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-presign-$(randstring(8)).json" + test_data = "{\"presign\": true, \"test\": \"$(randstring(12))\"}" + put_file(storage, test_key, test_data) + + url = s3_sign_url(storage, test_key, 300) + @test occursin("https://s3-api.yiem.cc/testbucket/", url) + + delete_file(storage, test_key) + end + end