diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..afa2dd6 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,27 @@ +# 1. Garage Web Admin UI +garage-ui.yiem.cc { + reverse_proxy 192.168.88.106:3909 +} + +# 2. Garage S3 API (Path-Style & Single Domain) +s3-api.yiem.cc { + reverse_proxy 192.168.88.106:3900 { + header_up Host {host} + } +} + +# 3. Garage S3 WEB (Path-Style & Single Domain) +s3-web.yiem.cc { + # /bucket -> /bucket/ + @bucket_root path_regexp bucket_root ^/([^/]+)$ + redir @bucket_root /{http.regexp.bucket_root.1}/ 301 + + # /bucket/... -> Garage + @bucket path_regexp bucket ^/([^/]+)(/.*)?$ + + rewrite @bucket {http.regexp.bucket.2} + + reverse_proxy 192.168.88.106:3902 { + header_up Host {http.regexp.bucket.1}.s3-web.yiem.cc + } +} diff --git a/Manifest.toml b/Manifest.toml index 7ff0445..0d17d24 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.7" manifest_format = "2.0" -project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044" +project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2" [[deps.AWS]] deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"] diff --git a/Project.toml b/Project.toml index f074c5a..0b22b6a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,11 +1,12 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" version = "0.6.9" -authors = ["tonaerospace "] +authors = ["tonaerospace storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") -julia> put_file1 = put_file(storage1) -julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}") +julia> s3PutFile1 = s3PutFile(storage1) +julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> downloadUrl.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" @@ -146,10 +152,10 @@ julia> downloadUrl.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ -mutable struct put_file +mutable struct s3PutFile storage::GarageStorage end -function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}} +function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) println("Successfully uploaded: ", key) @@ -178,7 +184,7 @@ end # Examples ```julia -julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}") +julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> result.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" @@ -186,7 +192,7 @@ julia> result.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ -function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} +function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(storage.config, storage.bucket, key, data) println("Successfully uploaded: ", key) @@ -213,11 +219,11 @@ end # Examples ```julia -julia> data = String(get_file(storage, "test.json")) +julia> data = String(s3GetFile(storage, "test.json")) "\"{\\\"status\\\": \\\"active\\\"}\"" ``` """ -function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} +function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} try return AWSS3.s3_get(storage.config, storage.bucket, key) catch @@ -236,14 +242,14 @@ end # Examples ```julia -julia> keys = list_files(storage) +julia> keys = s3ListFiles(storage) 3-element Vector{String}: "test1.json" "test2.json" "users-1002.json" ``` """ -function list_files(storage::GarageStorage) +function s3ListFiles(storage::GarageStorage) # Use delimiter="" to get all individual keys (no grouping by "directory") objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="") return [obj["Key"] for obj in objects] @@ -259,13 +265,196 @@ end # Examples ```julia -julia> delete_file(storage, "test.json") +julia> s3DeleteFile(storage, "test.json") ``` """ -function delete_file(storage::GarageStorage, key::String) +function s3DeleteFile(storage::GarageStorage, key::String) AWSS3.s3_delete(storage.config, storage.bucket, key) end +""" + s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String + +Generate a pre-signed URL for an object in the Garage S3 bucket. + +A pre-signed URL grants temporary access without AWS credentials. Signature is computed +using AWS Signature Version 4 (HMAC-SHA256). + +# Arguments +- `storage::GarageStorage`: Storage client from `GarageStorage(endpoint, key, secret, bucket)`. +- `key::String`: Object key. Supports nested paths like `"folder/subfolder/file.txt"`. +- `seconds::Int`: URL validity in seconds. Default: 3600. + +# Keyword Arguments +- `verb::String = "GET"`: HTTP method (`"GET"` downloads, `"PUT"` uploads). +- `content_type::String = "application/octet-stream"`: Required `Content-Type` header. + For PUT, set to the file's MIME type (`"image/png"`, `"application/json"`, etc.). +- `signature_version::String = "v4"`: Always use `"v4"` (v2 is deprecated). + +# Return +`String`: The pre-signed URL. + +# Examples + +```julia +using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile + +storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "sommpanion" +) + +# Upload — PUT with curl +url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg") +# curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' + +# Download — GET with curl +url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET") +# curl → returns file content + +# Upload JSON +url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json") +# curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' + +# Delete +url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE") +# curl -X DELETE url + +# Multiple uploads (chat app) +urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300; + verb="PUT", content_type="image/png") + for _ in 1:10] + +# Signature differs by verb and content_type +url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET") +url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain") +``` + +# Notes +- URLs are valid until `seconds` expires; they can be used **multiple times** (not one-time). +- Server clocks must be within ~5 minutes of the Garage server (UTC time in signature). +- PUT uploads require the client's `Content-Type` to match `content_type`. +- For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`. +""" +function s3SignUrl( + storage::GarageStorage, + key::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + signature_version::String="v4", +)::String + config = storage.config + creds = config.credentials + bucket = storage.bucket + region = config.region + endpoint = strip(config.endpoint, '/') + scheme = "AWS4" + algorithm = "HMAC-SHA256" + terminator = "aws4_request" + service = "s3" + + now_datetime = Dates.now(Dates.UTC) + datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") + date_stamp = Dates.format(now_datetime, "YYYYmmdd") + + scope = "$date_stamp/$region/$service/$terminator" + + headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host) + if verb == "PUT" + headers["Content-Type"] = content_type + end + sort!(headers; by=name -> lowercase(name)) + canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") + + query = OrderedDict{String,String}( + "X-Amz-Expires" => string(seconds), + "X-Amz-Algorithm" => "$scheme-$algorithm", + "X-Amz-Credential" => "$(creds.access_key_id)/$scope", + "X-Amz-Date" => datetime_stamp, + "X-Amz-SignedHeaders" => canonical_header_names, + ) + if !isempty(creds.token) + query["X-Amz-Security-Token"] = creds.token + end + + canonical_headers = join( + map( + header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", + collect(headers), + ), + ) + + resource_path = "/$bucket/$key" + + # Build canonical query string: sort by key=value pair lexicographically, + # URL-encoding each key and value (matching Garage's uri_encode) + sorted_q = sort( + [URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)], + by=lowercase, + ) + canonical_query = join(sorted_q, "&") + + canonical_request = string( + "$verb\n", + resource_path, + "\n", + canonical_query, + "\n", + canonical_headers, + "\n", + canonical_header_names, + "\n", + "UNSIGNED-PAYLOAD", + ) + + string_to_sign = string( + "$scheme-$algorithm\n", + "$datetime_stamp\n", + "$scope\n", + bytes2hex(digest(MD_SHA256, canonical_request)), + ) + + key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key)) + key_date = digest(MD_SHA256, date_stamp, key_secret_bytes) + key_region = digest(MD_SHA256, region, key_date) + key_service = digest(MD_SHA256, service, key_region) + key_signing = digest(MD_SHA256, terminator, key_service) + signature = digest(MD_SHA256, string_to_sign, key_signing) + + query["X-Amz-Signature"] = bytes2hex(signature) + + query_string = URIs.escapeuri(query) + return "$(endpoint)/$bucket/$key?$query_string" +end + +""" Generate a pre-signed URL using the Garage S3 web endpoint (s3-web). + +Useful for direct browser access — URLs look like +`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...` + +Arguments and keyword arguments are the same as `s3SignUrl`. +""" +function s3SignUrlWeb( + storage::GarageStorage, + key::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + signature_version::String="v4", +)::String + web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web") + web_creds = SimpleCredentials( + storage.config.credentials.access_key_id, + storage.config.credentials.secret_key, + storage.config.credentials.token, + ) + web_config = GarageConfig(web_endpoint, storage.config.region, web_creds) + web_storage = GarageStorage(web_config, storage.bucket) + return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version) +end # =================================================================== # 3. Bucket Lifecycle Configuration @@ -323,10 +512,10 @@ end # Example ```julia rule = LifecycleExpiration("uploads/", 30) -set_lifecycle_expiration(storage, rule) +s3SetLifecycleExpiration(storage, rule) ``` """ -function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration) +function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration) xml_body = build_lifecycle_xml(rule) args = Dict{String,Any}("body" => xml_body) return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config) @@ -342,10 +531,10 @@ end # Example ```julia -config = get_lifecycle(storage) +config = s3GetLifecycle(storage) ``` """ -function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} +function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} try return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config) catch e @@ -366,10 +555,10 @@ end # Example ```julia -delete_lifecycle(storage) +s3DeleteLifecycle(storage) ``` """ -function delete_lifecycle(storage::GarageStorage) +function s3DeleteLifecycle(storage::GarageStorage) return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config) end diff --git a/test/etc.jl b/test/etc.jl index e69de29..816297d 100644 --- a/test/etc.jl +++ b/test/etc.jl @@ -0,0 +1,130 @@ +curl -X PUT -H "Content-Type: application/json" \ + -d '{"test": "data"}' \ + "https://s3-api.yiem.cc/sommpanion/pcimage.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T103745Z&X-Amz-Expires=300&X-Amz-Security-Token=&X-Amz-SignedHeaders=host&X-Amz-Signature=0e195a48f824a2f8ffcb06461396cd6a8bddd5b61d93e9006f947ae10ecf525f" + + + + + + + + + +curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ + -H 'Host: s3-api.yiem.cc' \ + -H 'Content-Type: image/jpeg' \ + "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T030315Z&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=1107a31f36f8b8ceb9663d69b9a7be7733df14d019e700e750944cbdad50df5d" + + + + + + + + + + + + + + + + +read +./garage.toml +./Caddyfile +./src/garageS3.jl + + + +julia> url = GeneralUtils.s3_sign_url(storage1, "photo.jpg", 36000; verb="PUT", content_type="image/jpeg") +"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T145004Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=92cd634dbb0f66db4b8b039ef5939c138203de260e8342fe27ef78979d90a824" + +--- + + + + + + + + + + + +ton@server12vm-dev:~/docker-apps/sommpanion/etc/AWS.jl$ +curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ + -H 'Host: s3-api.yiem.cc' \ + -H 'Content-Type: image/jpeg' \ + "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02" + +Note: Unnecessary use of -X or --request, PUT is already inferred. +* Trying 110.77.160.154:443... +* Connected to s3-api.yiem.cc (110.77.160.154) port 443 (#0) +* ALPN: offers h2,http/1.1 +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +* CAfile: /etc/ssl/certs/ca-certificates.crt +* CApath: /etc/ssl/certs +* TLSv1.3 (IN), TLS handshake, Server hello (2): +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +* TLSv1.3 (IN), TLS handshake, Certificate (11): +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +* TLSv1.3 (IN), TLS handshake, Finished (20): +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +* TLSv1.3 (OUT), TLS handshake, Finished (20): +* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256 +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=s3-api.yiem.cc +* start date: Aug 27 04:27:02 2026 GMT +* expire date: Nov 25 04:27:01 2026 GMT +* subjectAltName: host "s3-api.yiem.cc" matched cert's "s3-api.yiem.cc" +* issuer: C=US; O=Let's Encrypt; CN=YE2 +* SSL certificate verify ok. +* using HTTP/2 +* h2h3 [:method: PUT] +* h2h3 [:path: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02] +* h2h3 [:scheme: https] +* h2h3 [:authority: s3-api.yiem.cc] +* h2h3 [user-agent: curl/7.88.1] +* h2h3 [accept: */*] +* h2h3 [content-type: image/jpeg] +* h2h3 [content-length: 439490] +* Using Stream ID: 1 (easy handle 0x559a08824c80) +> PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 HTTP/2 +> Host: s3-api.yiem.cc +> user-agent: curl/7.88.1 +> accept: */* +> content-type: image/jpeg +> content-length: 439490 +> +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +* We are completely uploaded and fine +< HTTP/2 403 +< access-control-allow-origin: * +< alt-svc: h3=":443"; ma=2592000 +< content-type: application/xml +< date: Fri, 04 Sep 2026 15:23:13 GMT +< via: 1.1 Caddy +< content-length: 190 +< +* Connection #0 to host s3-api.yiem.cc left intact +AccessDeniedForbidden: Invalid signature/sommpanion/photo.jpggarage + +2026-09-04T15:23:13.796037Z INFO garage_api_common::generic_server: 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 +2026-09-04T15:23:13.796077Z DEBUG garage_api_common::generic_server: Request { method: PUT, uri: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02, version: HTTP/1.1, headers: {"host": "s3-api.yiem.cc", "user-agent": "curl/7.88.1", "content-length": "439490", "accept": "*/*", "content-type": "image/jpeg", "via": "2.0 Caddy", "x-forwarded-for": "110.77.160.154", "x-forwarded-host": "s3-api.yiem.cc", "x-forwarded-proto": "https", "accept-encoding": "gzip"}, body: Body(Streaming) } +2026-09-04T15:23:13.796181Z DEBUG garage_api_common::generic_server: Endpoint: PutObject +2026-09-04T15:23:13.796461Z INFO garage_api_common::generic_server: error 403 Forbidden, Forbidden: Invalid signature in response to 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 + + + + + + + + + + + + + diff --git a/test/runtests.jl b/test/runtests.jl index 8ef2cf7..e19cd29 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,8 +2,7 @@ using Test using GeneralUtils using GeneralUtils.garageS3 using AWS -using Random: randstring -using UUIDs: uuid4 +using URIs @testset "garageS3.jl" begin @@ -142,7 +141,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-$(randstring(8)).json" + test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" # Test put_file(storage, key, data) - successful upload @@ -165,7 +164,7 @@ using UUIDs: uuid4 @test test_key in keys # Test put_file callable struct - test_key2 = "testrunner-callable-$(randstring(8)).json" + test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json" callable_result = put_file(storage)(test_key2, "callable-data") @test callable_result !== nothing @test callable_result.api != "" @@ -194,7 +193,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "single/slash/test-$(randstring(8)).json" + test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"level\": 1, \"nested\": true}" # Upload nested key @@ -227,7 +226,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "deep/nested/folder/subfolder/test-$(randstring(8)).json" + test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}" # Upload deeply nested key @@ -259,7 +258,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "callable/nested/test-$(randstring(8)).json" + test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"callable\": true, \"nested\": true}" uploader = put_file(storage) @@ -281,7 +280,7 @@ using UUIDs: uuid4 "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - result = get_file(storage, "nonexistent-key-$(randstring(8)).json") + result = get_file(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json") @test result === nothing end @@ -293,7 +292,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-binary-$(randstring(8)).bin" + test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin" binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] result = put_file(storage, test_key, binary_data) @@ -314,7 +313,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-list-$(randstring(8)).txt" + test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt" put_file(storage, test_key, "list-test-data") keys = list_files(storage) @@ -332,7 +331,7 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-url-$(randstring(8)).json" + test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json" result = put_file(storage, test_key, "url-test") @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") @@ -460,4 +459,129 @@ using UUIDs: uuid4 delete_lifecycle(storage) end + # --- Pre-signed URL tests --- + + @testset "s3_sign_url returns a string" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "test.json", 3600) + @test url isa String + @test !isempty(url) + end + + @testset "s3_sign_url URL format contains correct endpoint, bucket, key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "mykey.json", 1800) + @test occursin("https://s3-api.yiem.cc", url) + @test occursin("/testbucket/mykey.json?", url) + end + + @testset "s3_sign_url contains required query parameters" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "test.json", 3600) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url) + @test occursin("X-Amz-Expires=3600", url) + @test occursin("X-Amz-Credential=", url) + @test occursin("X-Amz-Date=", url) + @test occursin("X-Amz-SignedHeaders=", url) + @test occursin("X-Amz-Signature=", url) + end + + @testset "s3_sign_url with custom expiry duration" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3_sign_url(storage, "test.json", 600) + @test occursin("X-Amz-Expires=600", url) + end + + @testset "s3_sign_url with GET verb produces different signature than default" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url_get = s3_sign_url(storage, "test.json", 3600; verb="GET") + url_default = s3_sign_url(storage, "test.json", 3600) + # GET verb should produce a valid signature + @test occursin("X-Amz-Signature=", url_get) + @test occursin("X-Amz-Signature=", url_default) + # Both URLs should have the same structure but potentially different signatures + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get) + end + + @testset "s3_sign_url with PUT verb and content_type" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url_put = s3_sign_url(storage, "upload.json", 300; verb="PUT", content_type="application/json") + url_get = s3_sign_url(storage, "upload.json", 300; verb="GET") + # PUT and GET should produce different signatures (different canonical request) + @test occursin("X-Amz-Signature=", url_put) + @test occursin("X-Amz-Signature=", url_get) + end + + @testset "s3_sign_url with special characters in key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + test_key = "folder/sub folder/file with spaces.json" + url = s3_sign_url(storage, test_key, 3600) + @test occursin("testbucket/", url) + end + + @testset "s3_sign_url with deeply nested key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + test_key = "a/b/c/d/e/deep-file.json" + url = s3_sign_url(storage, test_key, 3600) + @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url) + end + + @testset "integration: pre-signed URL can download existing file" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}" + put_file(storage, test_key, test_data) + + url = s3_sign_url(storage, test_key, 300) + @test occursin("https://s3-api.yiem.cc/testbucket/", url) + + delete_file(storage, test_key) + end + end diff --git a/test/winebottle.jpg b/test/winebottle.jpg new file mode 100644 index 0000000..7de5a42 Binary files /dev/null and b/test/winebottle.jpg differ