update

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.12.7"
 manifest_format = "2.0"
-project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044"
+project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2"
 
 [[deps.AWS]]
 deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"]

Project.toml

@@ -1,11 +1,12 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
 version = "0.6.9"
-authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
+authors = ["tonaerospace <tonaerospace.etc@gmail.com"]
 
 [deps]
 AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
 AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
@@ -15,6 +16,7 @@ Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
+MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
 NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -28,10 +30,12 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [compat]
 AWS = "1.99.0"
 AWSS3 = "0.11.6"
+Base64 = "1.11.0"
 Graphs = "1.14.0"
 HTTP = "2.5.0 - 2.9.9"
 JSON = "1.3.0 - 1.9.9"
 LibPQ = "1.18.0"
+MbedTLS = "1.1.10"
 NATS = "0.1.0"
 OrderedCollections = "2.0.1"
 Revise = "3.13.2"

src/garageS3.jl

@@ -3,11 +3,17 @@ module garageS3
 export
     LifecycleExpiration, build_lifecycle_xml,
     GarageStorage, put_file, get_file, list_files, delete_file,
-    set_lifecycle_expiration, get_lifecycle, delete_lifecycle
+    set_lifecycle_expiration, get_lifecycle, delete_lifecycle,
+    s3_sign_url
 
 using AWS 
 @service S3
 using AWSS3
+using Dates: Dates
+using URIs: URIs
+using OrderedCollections: OrderedDict
+using MbedTLS: digest, MD_SHA256
+using Base64: bytes2hex
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 # Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
@@ -266,6 +272,262 @@ function delete_file(storage::GarageStorage, key::String)
     AWSS3.s3_delete(storage.config, storage.bucket, key)
 end
 
+"""
+    s3_sign_url(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String
+
+Generate a pre-signed URL for an object in the Garage S3 bucket.
+
+A pre-signed URL is a signed URL that grants temporary access to an S3 object without
+requiring AWS credentials. The URL expires after the specified number of seconds.
+The signature is computed using AWS Signature Version 4 (HMAC-SHA256).
+
+# Why use pre-signed URLs?
+
+Pre-signed URLs let you grant temporary, scoped access to objects without sharing
+your API credentials. Typical use cases:
+
+- **Direct browser downloads**: Return the URL to a frontend so users can download
+  files without going through your server.
+- **Direct browser uploads**: Generate a PUT pre-signed URL, pass it to the frontend,
+  and let the user upload a file directly to S3 with a `curl` or `fetch` call.
+- **Temporary sharing**: Send an expiring link to someone who should not have
+  permanent access to the object.
+- **Server-side download without round-trip**: Instead of downloading the file to
+  your server and re-uploading it elsewhere, pass the pre-signed URL to another
+  service.
+
+# How it works
+
+The function computes an HMAC-SHA256 signature over a canonical request that
+includes the HTTP verb, the object path, the query parameters, and the host header.
+The signature is appended to the URL as `X-Amz-Signature`. Garage (and any S3
+compatible service) verifies the signature before serving the object.
+
+# URL format
+https://s3-api.your-garage-instance.com/bucket/key?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...&X-Amz-Date=...&X-Amz-Expires=...&X-Amz-SignedHeaders=...&X-Amz-Signature=...
+
+# Arguments
+- `storage::GarageStorage`
+  A storage client created with `GarageStorage(endpoint, key, secret, bucket)`.
+- `key::String`
+  The object key (file name) in the bucket. Supports nested paths like
+  `"folder/subfolder/file.txt"`.
+- `seconds::Int`
+  How long the URL remains valid, in seconds. Default: 3600 (1 hour).
+
+# Keyword Arguments
+- `verb::String = "GET"`
+  The HTTP method the URL permits. Use `"GET"` for downloads (the default) or
+  `"PUT"` for uploads. Other values like `"DELETE"` are also supported.
+- `content_type::String = "application/octet-stream"`
+  The `Content-Type` header value that must match when the URL is used.
+  For PUT uploads, set this to the actual MIME type of the file you plan to
+  upload (e.g. `"text/plain"`, `"application/json"`, `"image/png"`).
+- `signature_version::String = "v4"`
+  Signature algorithm version. Always use `"v4"` — version 2 is deprecated
+  and not supported by many S3-compatible services.
+
+# Return
+- `String`: The full pre-signed URL, ready to use with `curl`, a browser, or any
+  HTTP client.
+
+# Examples
+
+### Download link (GET — default)
+```julia
+storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket")
+
+# 1-hour download link
+url = s3_sign_url(storage, "documents/report.pdf", 3600)
+println("Download link: ", url)
+
+# Use in a browser — opens the file directly
+# > open(url)
+
+# Use with curl from the terminal
+# > curl -L "url" -o report.pdf
+```
+
+### Upload link (PUT)
+```julia
+# Generate a PUT URL valid for 10 minutes, accepting text/plain uploads
+url = s3_sign_url(storage, "uploads/user-photo.png", 600;
+                  verb="PUT", content_type="image/png")
+
+# The user (or your frontend) uploads a file like this:
+# > curl -X PUT -H "Content-Type: image/png" --data @photo.png "url"
+```
+
+### Embedding in HTML
+```julia
+url = s3_sign_url(storage, "shared/shared-file.zip", 86400)  # 24 hours
+# Use the URL in HTML: <a href="url">Download shared-file.zip (valid 24 hours)</a>
+```
+
+### Verification — signature differs by verb and content-type
+```julia
+url_get = s3_sign_url(storage, "file.txt", 3600; verb="GET")
+url_put = s3_sign_url(storage, "file.txt", 3600; verb="PUT", content_type="text/plain")
+# url_get and url_put have different X-Amz-Signature values because the
+# canonical request (and thus the HMAC) is different.
+```
+
+# Notes
+- The pre-signed URL is **not encrypted** — anyone with the URL can access the
+  object until it expires. Treat the URL like a temporary password.
+- The signature includes the current UTC time (`X-Amz-Date`), so both the
+  generating machine and the Garage server should have reasonably synchronized
+  clocks (± a few minutes tolerance).
+- Only `v4` signature is implemented. Garage supports v4 natively.
+- For cross-origin (CORS) uploads from a browser, you also need the Garage bucket
+  to have CORS rules that allow the `Origin` and `Method` headers.
+
+# s3_sign_url vs get_file — when to use which
+
+| Scenario | Use |
+|---|---|
+| Your server needs to read/write an object | `get_file(storage, key)`, `put_file(storage, key, data)` |
+| A browser or external client needs to download an object | `s3_sign_url(storage, key, 3600; verb="GET")` |
+| A browser or external client needs to upload a file | `s3_sign_url(storage, key, 600; verb="PUT", content_type="image/png")` |
+| You need to pass a download link to a third party | `s3_sign_url(storage, key, 86400)` — no credentials shared |
+
+`get_file` / `put_file` use your API credentials on every call.
+`s3_sign_url` embeds a time-limited signature in the URL — no credentials needed.
+
+# Complete upload workflow example
+
+```julia
+# 1. Generate a PUT pre-signed URL on your server
+url = s3_sign_url(storage, "avatars/user-12345.png", 300;
+                  verb="PUT", content_type="image/png")
+
+# 2. Send `url` to your frontend (JSON response, React state, etc.)
+
+# 3. Frontend uploads directly to Garage:
+#    fetch(url, { method: "PUT", headers: {"Content-Type": "image/png"}, body: fileBlob })
+#    or:
+#    curl -X PUT -H "Content-Type: image/png" --data @photo.png url
+
+# 4. Optional — verify the upload succeeded
+#    data = get_file(storage, "avatars/user-12345.png")
+```
+
+# Complete download workflow example
+
+```julia
+# 1. Generate a GET pre-signed URL
+url = s3_sign_url(storage, "reports/quarterly.pdf", 7200)  # 2 hours
+
+# 2a. Redirect user to the URL (S3 returns 302)
+#     return redirect(url)
+
+# 2b. Embed in HTML link
+#     <a href="url" download>Download quarterly.pdf</a>
+
+# 2c. Use with curl
+#     curl -L url -o quarterly.pdf
+```
+
+# Query parameters in the generated URL
+
+| Parameter | Purpose |
+|---|---|
+| `X-Amz-Algorithm` | Must be `AWS4-HMAC-SHA256` |
+| `X-Amz-Credential` | Your access key ID + date + region + service + request token |
+| `X-Amz-Date` | Timestamp when the URL was generated (UTC, ISO 8601 format) |
+| `X-Amz-Expires` | Number of seconds until expiration |
+| `X-Amz-Security-Token` | Optional session token (empty if not provided) |
+| `X-Amz-SignedHeaders` | Semicolon-separated list of signed headers (currently only `host`) |
+| `X-Amz-Signature` | The HMAC-SHA256 signature — the core authorization token |
+
+# Common pitfalls
+
+- **Clock skew**: If the server and Garage are more than ~5 minutes out of sync,
+  the signature will be rejected.
+- **Content-Type mismatch**: When using a PUT URL, the `Content-Type` sent by the
+  client must match the `content_type` you passed to `s3_sign_url`.
+- **URL encoding**: The key is URL-encoded in the path. Keys with spaces or special
+  characters work correctly, but do not double-encode the returned URL.
+- **Single use**: Pre-signed URLs can be used multiple times until expiration;
+  they are not one-time tokens.
+"""
+function s3_sign_url(
+    storage::GarageStorage,
+    key::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    signature_version::String="v4",
+)::String
+    config = storage.config
+    creds = config.credentials
+    bucket = storage.bucket
+    region = config.region
+    endpoint = strip(config.endpoint, '/')
+    scheme = "AWS4"
+    algorithm = "HMAC-SHA256"
+    terminator = "aws4_request"
+    service = "s3"
+
+    now_datetime = Dates.now(Dates.UTC)
+    datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
+    date_stamp = Dates.format(now_datetime, "YYYYmmdd")
+
+    scope = "$date_stamp/$region/$service/$terminator"
+
+    headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host)
+    sort!(headers; by=name -> lowercase(name))
+    canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
+
+    query = OrderedDict{String,String}(
+        "X-Amz-Expires" => string(seconds),
+        "X-Amz-Algorithm" => "$scheme-$algorithm",
+        "X-Amz-Credential" => "$(creds.access_key_id)/$scope",
+        "X-Amz-Date" => datetime_stamp,
+        "X-Amz-Security-Token" => isempty(creds.token) ? "" : creds.token,
+        "X-Amz-SignedHeaders" => canonical_header_names,
+    )
+
+    sort!(query; by=name -> lowercase(name))
+
+    canonical_headers = join(
+        map(
+            header -> "$(lowercase(header.first)):$(lowercase(header.second))\n",
+            collect(headers),
+        ),
+    )
+
+    resource_path = "/$bucket/$key"
+
+    canonical_request = string(
+        "$verb\n",
+        "$(URIs.escapeuri(resource_path))\n",
+        "$(URIs.escapeuri(query))\n",
+        "$canonical_headers\n",
+        "$canonical_header_names\n",
+        "UNSIGNED-PAYLOAD",
+    )
+
+    string_to_sign = string(
+        "$scheme-$algorithm\n",
+        "$datetime_stamp\n",
+        "$scope\n",
+        bytes2hex(digest(MD_SHA256, canonical_request)),
+    )
+
+    key_secret = string(scheme, creds.secret_key)
+    key_date = digest(MD_SHA256, date_stamp, key_secret)
+    key_region = digest(MD_SHA256, region, key_date)
+    key_service = digest(MD_SHA256, service, key_region)
+    key_signing = digest(MD_SHA256, terminator, key_service)
+    signature = digest(MD_SHA256, string_to_sign, key_signing)
+
+    query["X-Amz-Signature"] = bytes2hex(signature)
+
+    query_string = URIs.escapeuri(query)
+    return "$(endpoint)/$bucket/$key?$query_string"
+end
+
 
 # ===================================================================
 # 3. Bucket Lifecycle Configuration

test/runtests.jl

@@ -4,6 +4,7 @@ using GeneralUtils.garageS3
 using AWS
 using Random: randstring
 using UUIDs: uuid4
+using URIs
 
 @testset "garageS3.jl" begin
 
@@ -460,4 +461,129 @@ using UUIDs: uuid4
         delete_lifecycle(storage)
     end
 
+    # --- Pre-signed URL tests ---
+
+    @testset "s3_sign_url returns a string" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "test.json", 3600)
+        @test url isa String
+        @test !isempty(url)
+    end
+
+    @testset "s3_sign_url URL format contains correct endpoint, bucket, key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "mykey.json", 1800)
+        @test occursin("https://s3-api.yiem.cc", url)
+        @test occursin("/testbucket/mykey.json?", url)
+    end
+
+    @testset "s3_sign_url contains required query parameters" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "test.json", 3600)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url)
+        @test occursin("X-Amz-Expires=3600", url)
+        @test occursin("X-Amz-Credential=", url)
+        @test occursin("X-Amz-Date=", url)
+        @test occursin("X-Amz-SignedHeaders=", url)
+        @test occursin("X-Amz-Signature=", url)
+    end
+
+    @testset "s3_sign_url with custom expiry duration" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3_sign_url(storage, "test.json", 600)
+        @test occursin("X-Amz-Expires=600", url)
+    end
+
+    @testset "s3_sign_url with GET verb produces different signature than default" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url_get = s3_sign_url(storage, "test.json", 3600; verb="GET")
+        url_default = s3_sign_url(storage, "test.json", 3600)
+        # GET verb should produce a valid signature
+        @test occursin("X-Amz-Signature=", url_get)
+        @test occursin("X-Amz-Signature=", url_default)
+        # Both URLs should have the same structure but potentially different signatures
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get)
+    end
+
+    @testset "s3_sign_url with PUT verb and content_type" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url_put = s3_sign_url(storage, "upload.json", 300; verb="PUT", content_type="application/json")
+        url_get = s3_sign_url(storage, "upload.json", 300; verb="GET")
+        # PUT and GET should produce different signatures (different canonical request)
+        @test occursin("X-Amz-Signature=", url_put)
+        @test occursin("X-Amz-Signature=", url_get)
+    end
+
+    @testset "s3_sign_url with special characters in key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        test_key = "folder/sub folder/file with spaces.json"
+        url = s3_sign_url(storage, test_key, 3600)
+        @test occursin("testbucket/", url)
+    end
+
+    @testset "s3_sign_url with deeply nested key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        test_key = "a/b/c/d/e/deep-file.json"
+        url = s3_sign_url(storage, test_key, 3600)
+        @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url)
+    end
+
+    @testset "integration: pre-signed URL can download existing file" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-presign-$(randstring(8)).json"
+        test_data = "{\"presign\": true, \"test\": \"$(randstring(12))\"}"
+        put_file(storage, test_key, test_data)
+
+        url = s3_sign_url(storage, test_key, 300)
+        @test occursin("https://s3-api.yiem.cc/testbucket/", url)
+
+        delete_file(storage, test_key)
+    end
+
 end