ton/GeneralUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

V0.6.10 #16

Merged
ton merged 8 commits from v0.6.10 into main 2026-09-05 14:16:56 +00:00
Conversation 0 Commits 8 Files Changed 11
+395 -3
4 changed files with 395 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 2827d22616 - Show all commits
Manifest.toml
+1 -1
View File
@@ -2,7 +2,7 @@
julia_version = "1.12.7"
manifest_format = "2.0"
project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044"
project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2"
[[deps.AWS]]
deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"]
Project.toml
+5 -1
View File
@@ -1,11 +1,12 @@
name = "GeneralUtils"
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
version = "0.6.9"
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
authors = ["tonaerospace <tonaerospace.etc@gmail.com"]
[deps]
AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
@@ -15,6 +16,7 @@ Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -28,10 +30,12 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[compat]
AWS = "1.99.0"
AWSS3 = "0.11.6"
Base64 = "1.11.0"
Graphs = "1.14.0"
HTTP = "2.5.0 - 2.9.9"
JSON = "1.3.0 - 1.9.9"
LibPQ = "1.18.0"
MbedTLS = "1.1.10"
NATS = "0.1.0"
OrderedCollections = "2.0.1"
Revise = "3.13.2"
src/garageS3.jl
+263 -1
View File
@@ -3,11 +3,17 @@ module garageS3
export
LifecycleExpiration, build_lifecycle_xml,
GarageStorage, put_file, get_file, list_files, delete_file,
set_lifecycle_expiration, get_lifecycle, delete_lifecycle
set_lifecycle_expiration, get_lifecycle, delete_lifecycle,
s3_sign_url
using AWS
@service S3
using AWSS3
using Dates: Dates
using URIs: URIs
using OrderedCollections: OrderedDict
using MbedTLS: digest, MD_SHA256
using Base64: bytes2hex
# ---------------------------------------------- 100 --------------------------------------------- #
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
@@ -266,6 +272,262 @@ function delete_file(storage::GarageStorage, key::String)
AWSS3.s3_delete(storage.config, storage.bucket, key)
end
"""
s3_sign_url(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String
Generate a pre-signed URL for an object in the Garage S3 bucket.
A pre-signed URL is a signed URL that grants temporary access to an S3 object without
requiring AWS credentials. The URL expires after the specified number of seconds.
The signature is computed using AWS Signature Version 4 (HMAC-SHA256).
# Why use pre-signed URLs?
Pre-signed URLs let you grant temporary, scoped access to objects without sharing
your API credentials. Typical use cases:
- **Direct browser downloads**: Return the URL to a frontend so users can download
files without going through your server.
- **Direct browser uploads**: Generate a PUT pre-signed URL, pass it to the frontend,
and let the user upload a file directly to S3 with a `curl` or `fetch` call.
- **Temporary sharing**: Send an expiring link to someone who should not have
permanent access to the object.
- **Server-side download without round-trip**: Instead of downloading the file to
your server and re-uploading it elsewhere, pass the pre-signed URL to another
service.
# How it works
The function computes an HMAC-SHA256 signature over a canonical request that
includes the HTTP verb, the object path, the query parameters, and the host header.
The signature is appended to the URL as `X-Amz-Signature`. Garage (and any S3
compatible service) verifies the signature before serving the object.
# URL format
https://s3-api.your-garage-instance.com/bucket/key?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...&X-Amz-Date=...&X-Amz-Expires=...&X-Amz-SignedHeaders=...&X-Amz-Signature=...
# Arguments
- `storage::GarageStorage`
A storage client created with `GarageStorage(endpoint, key, secret, bucket)`.
- `key::String`
The object key (file name) in the bucket. Supports nested paths like
`"folder/subfolder/file.txt"`.
- `seconds::Int`
How long the URL remains valid, in seconds. Default: 3600 (1 hour).
# Keyword Arguments
- `verb::String = "GET"`
The HTTP method the URL permits. Use `"GET"` for downloads (the default) or
`"PUT"` for uploads. Other values like `"DELETE"` are also supported.
- `content_type::String = "application/octet-stream"`
The `Content-Type` header value that must match when the URL is used.
For PUT uploads, set this to the actual MIME type of the file you plan to
upload (e.g. `"text/plain"`, `"application/json"`, `"image/png"`).
- `signature_version::String = "v4"`
Signature algorithm version. Always use `"v4"` — version 2 is deprecated
and not supported by many S3-compatible services.
# Return
- `String`: The full pre-signed URL, ready to use with `curl`, a browser, or any
HTTP client.
# Examples
### Download link (GET — default)
```julia
storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket")
# 1-hour download link
url = s3_sign_url(storage, "documents/report.pdf", 3600)
println("Download link: ", url)
# Use in a browser — opens the file directly
# > open(url)
# Use with curl from the terminal
# > curl -L "url" -o report.pdf
```
### Upload link (PUT)
```julia
# Generate a PUT URL valid for 10 minutes, accepting text/plain uploads
url = s3_sign_url(storage, "uploads/user-photo.png", 600;
verb="PUT", content_type="image/png")
# The user (or your frontend) uploads a file like this:
# > curl -X PUT -H "Content-Type: image/png" --data @photo.png "url"
```
### Embedding in HTML
```julia
url = s3_sign_url(storage, "shared/shared-file.zip", 86400) # 24 hours
# Use the URL in HTML: <a href="url">Download shared-file.zip (valid 24 hours)</a>
```
### Verification — signature differs by verb and content-type
```julia
url_get = s3_sign_url(storage, "file.txt", 3600; verb="GET")
url_put = s3_sign_url(storage, "file.txt", 3600; verb="PUT", content_type="text/plain")
# url_get and url_put have different X-Amz-Signature values because the
# canonical request (and thus the HMAC) is different.
```
# Notes
- The pre-signed URL is **not encrypted** — anyone with the URL can access the
object until it expires. Treat the URL like a temporary password.
- The signature includes the current UTC time (`X-Amz-Date`), so both the
generating machine and the Garage server should have reasonably synchronized
clocks (± a few minutes tolerance).
- Only `v4` signature is implemented. Garage supports v4 natively.
- For cross-origin (CORS) uploads from a browser, you also need the Garage bucket
to have CORS rules that allow the `Origin` and `Method` headers.
# s3_sign_url vs get_file — when to use which
| Scenario | Use |
|---|---|
| Your server needs to read/write an object | `get_file(storage, key)`, `put_file(storage, key, data)` |
| A browser or external client needs to download an object | `s3_sign_url(storage, key, 3600; verb="GET")` |
| A browser or external client needs to upload a file | `s3_sign_url(storage, key, 600; verb="PUT", content_type="image/png")` |
| You need to pass a download link to a third party | `s3_sign_url(storage, key, 86400)` — no credentials shared |
`get_file` / `put_file` use your API credentials on every call.
`s3_sign_url` embeds a time-limited signature in the URL — no credentials needed.
# Complete upload workflow example
```julia
# 1. Generate a PUT pre-signed URL on your server
url = s3_sign_url(storage, "avatars/user-12345.png", 300;
verb="PUT", content_type="image/png")
# 2. Send `url` to your frontend (JSON response, React state, etc.)
# 3. Frontend uploads directly to Garage:
# fetch(url, { method: "PUT", headers: {"Content-Type": "image/png"}, body: fileBlob })
# or:
# curl -X PUT -H "Content-Type: image/png" --data @photo.png url
# 4. Optional — verify the upload succeeded
# data = get_file(storage, "avatars/user-12345.png")
```
# Complete download workflow example
```julia
# 1. Generate a GET pre-signed URL
url = s3_sign_url(storage, "reports/quarterly.pdf", 7200) # 2 hours
# 2a. Redirect user to the URL (S3 returns 302)
# return redirect(url)
# 2b. Embed in HTML link
# <a href="url" download>Download quarterly.pdf</a>
# 2c. Use with curl
# curl -L url -o quarterly.pdf
```
# Query parameters in the generated URL
| Parameter | Purpose |
|---|---|
| `X-Amz-Algorithm` | Must be `AWS4-HMAC-SHA256` |
| `X-Amz-Credential` | Your access key ID + date + region + service + request token |
| `X-Amz-Date` | Timestamp when the URL was generated (UTC, ISO 8601 format) |
| `X-Amz-Expires` | Number of seconds until expiration |
| `X-Amz-Security-Token` | Optional session token (empty if not provided) |
| `X-Amz-SignedHeaders` | Semicolon-separated list of signed headers (currently only `host`) |
| `X-Amz-Signature` | The HMAC-SHA256 signature — the core authorization token |
# Common pitfalls
- **Clock skew**: If the server and Garage are more than ~5 minutes out of sync,
the signature will be rejected.
- **Content-Type mismatch**: When using a PUT URL, the `Content-Type` sent by the
client must match the `content_type` you passed to `s3_sign_url`.
- **URL encoding**: The key is URL-encoded in the path. Keys with spaces or special
characters work correctly, but do not double-encode the returned URL.
- **Single use**: Pre-signed URLs can be used multiple times until expiration;
they are not one-time tokens.
"""
function s3_sign_url(
storage::GarageStorage,
key::String,
seconds::Int=3600;
verb::String="GET",
content_type::String="application/octet-stream",
signature_version::String="v4",
)::String
config = storage.config
creds = config.credentials
bucket = storage.bucket
region = config.region
endpoint = strip(config.endpoint, '/')
scheme = "AWS4"
algorithm = "HMAC-SHA256"
terminator = "aws4_request"
service = "s3"
now_datetime = Dates.now(Dates.UTC)
datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
date_stamp = Dates.format(now_datetime, "YYYYmmdd")
scope = "$date_stamp/$region/$service/$terminator"
headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host)
sort!(headers; by=name -> lowercase(name))
canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
query = OrderedDict{String,String}(
"X-Amz-Expires" => string(seconds),
"X-Amz-Algorithm" => "$scheme-$algorithm",
"X-Amz-Credential" => "$(creds.access_key_id)/$scope",
"X-Amz-Date" => datetime_stamp,
"X-Amz-Security-Token" => isempty(creds.token) ? "" : creds.token,
"X-Amz-SignedHeaders" => canonical_header_names,
)
sort!(query; by=name -> lowercase(name))
canonical_headers = join(
map(
header -> "$(lowercase(header.first)):$(lowercase(header.second))\n",
collect(headers),
),
)
resource_path = "/$bucket/$key"
canonical_request = string(
"$verb\n",
"$(URIs.escapeuri(resource_path))\n",
"$(URIs.escapeuri(query))\n",
"$canonical_headers\n",
"$canonical_header_names\n",
"UNSIGNED-PAYLOAD",
)
string_to_sign = string(
"$scheme-$algorithm\n",
"$datetime_stamp\n",
"$scope\n",
bytes2hex(digest(MD_SHA256, canonical_request)),
)
key_secret = string(scheme, creds.secret_key)
key_date = digest(MD_SHA256, date_stamp, key_secret)
key_region = digest(MD_SHA256, region, key_date)
key_service = digest(MD_SHA256, service, key_region)
key_signing = digest(MD_SHA256, terminator, key_service)
signature = digest(MD_SHA256, string_to_sign, key_signing)
query["X-Amz-Signature"] = bytes2hex(signature)
query_string = URIs.escapeuri(query)
return "$(endpoint)/$bucket/$key?$query_string"
end
# ===================================================================
# 3. Bucket Lifecycle Configuration
test/runtests.jl
+126
View File
@@ -4,6 +4,7 @@ using GeneralUtils.garageS3
using AWS
using Random: randstring
using UUIDs: uuid4
using URIs
@testset "garageS3.jl" begin
@@ -460,4 +461,129 @@ using UUIDs: uuid4
delete_lifecycle(storage)
end
# --- Pre-signed URL tests ---
@testset "s3_sign_url returns a string" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
url = s3_sign_url(storage, "test.json", 3600)
@test url isa String
@test !isempty(url)
end
@testset "s3_sign_url URL format contains correct endpoint, bucket, key" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
url = s3_sign_url(storage, "mykey.json", 1800)
@test occursin("https://s3-api.yiem.cc", url)
@test occursin("/testbucket/mykey.json?", url)
end
@testset "s3_sign_url contains required query parameters" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
url = s3_sign_url(storage, "test.json", 3600)
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url)
@test occursin("X-Amz-Expires=3600", url)
@test occursin("X-Amz-Credential=", url)
@test occursin("X-Amz-Date=", url)
@test occursin("X-Amz-SignedHeaders=", url)
@test occursin("X-Amz-Signature=", url)
end
@testset "s3_sign_url with custom expiry duration" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
url = s3_sign_url(storage, "test.json", 600)
@test occursin("X-Amz-Expires=600", url)
end
@testset "s3_sign_url with GET verb produces different signature than default" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
url_get = s3_sign_url(storage, "test.json", 3600; verb="GET")
url_default = s3_sign_url(storage, "test.json", 3600)
# GET verb should produce a valid signature
@test occursin("X-Amz-Signature=", url_get)
@test occursin("X-Amz-Signature=", url_default)
# Both URLs should have the same structure but potentially different signatures
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get)
end
@testset "s3_sign_url with PUT verb and content_type" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
url_put = s3_sign_url(storage, "upload.json", 300; verb="PUT", content_type="application/json")
url_get = s3_sign_url(storage, "upload.json", 300; verb="GET")
# PUT and GET should produce different signatures (different canonical request)
@test occursin("X-Amz-Signature=", url_put)
@test occursin("X-Amz-Signature=", url_get)
end
@testset "s3_sign_url with special characters in key" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
test_key = "folder/sub folder/file with spaces.json"
url = s3_sign_url(storage, test_key, 3600)
@test occursin("testbucket/", url)
end
@testset "s3_sign_url with deeply nested key" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
test_key = "a/b/c/d/e/deep-file.json"
url = s3_sign_url(storage, test_key, 3600)
@test occursin("/testbucket/a/b/c/d/e/deep-file.json", url)
end
@testset "integration: pre-signed URL can download existing file" begin
storage = GarageStorage(
"https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"testbucket"
)
test_key = "testrunner-presign-$(randstring(8)).json"
test_data = "{\"presign\": true, \"test\": \"$(randstring(12))\"}"
put_file(storage, test_key, test_data)
url = s3_sign_url(storage, test_key, 300)
@test occursin("https://s3-api.yiem.cc/testbucket/", url)
delete_file(storage, test_key)
end
end