diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..a8004b1 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,41 @@ +# 1. Garage Web Admin UI +garage-ui.yiem.cc { + reverse_proxy 192.168.88.106:3909 +} + +# 2. Garage S3 API (Path-Style & Single Domain) +s3-api.yiem.cc { + # CORS: handle OPTIONS preflight + @options method OPTIONS { + header Access-Control-Request-Method * + } + handle @options { + respond * 204 + } + # CORS headers on all responses + header { + Access-Control-Allow-Origin "*" + Access-Control-Allow-Methods "GET, PUT, POST, DELETE, OPTIONS" + Access-Control-Allow-Headers "*" + Access-Control-Max-Age "86400" + } + reverse_proxy 192.168.88.106:3900 { + header_up Host {host} + } +} + +# 3. Garage S3 WEB (Path-Style & Single Domain) +s3-web.yiem.cc { + # /bucket -> /bucket/ + @bucket_root path_regexp bucket_root ^/([^/]+)$ + redir @bucket_root /{http.regexp.bucket_root.1}/ 301 + + # /bucket/... -> Garage + @bucket path_regexp bucket ^/([^/]+)(/.*)?$ + + rewrite @bucket {http.regexp.bucket.2} + + reverse_proxy 192.168.88.106:3902 { + header_up Host {http.regexp.bucket.1}.s3-web.yiem.cc + } +} diff --git a/Manifest.toml b/Manifest.toml index 7ff0445..0d17d24 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.7" manifest_format = "2.0" -project_hash = "92e6bf163bcece135b000f14dcd39dd92d77a044" +project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2" [[deps.AWS]] deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"] diff --git a/Project.toml b/Project.toml index f074c5a..88e4079 100644 --- a/Project.toml +++ b/Project.toml @@ -1,11 +1,12 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -version = "0.6.9" -authors = ["tonaerospace "] +version = "0.6.10" +authors = ["tonaerospace storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") -julia> put_file1 = put_file(storage1) -julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}") +julia> s3PutFile1 = s3PutFile(storage1) +julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> downloadUrl.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" @@ -146,10 +152,10 @@ julia> downloadUrl.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ -mutable struct put_file +mutable struct s3PutFile storage::GarageStorage end -function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}} +function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) println("Successfully uploaded: ", key) @@ -178,7 +184,7 @@ end # Examples ```julia -julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}") +julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}") Successfully uploaded: test.json julia> result.api "https://s3-api.yiem.cc/sommpanion-s3/test.json" @@ -186,7 +192,7 @@ julia> result.web "https://s3-web.yiem.cc/sommpanion-s3/test.json" ``` """ -function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} +function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(storage.config, storage.bucket, key, data) println("Successfully uploaded: ", key) @@ -213,13 +219,13 @@ end # Examples ```julia -julia> data = String(get_file(storage, "test.json")) +julia> data = String(s3GetFile(storage, "test.json")) "\"{\\\"status\\\": \\\"active\\\"}\"" ``` """ -function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} +function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} try - return AWSS3.s3_get(storage.config, storage.bucket, key) + return AWSS3.s3_get(storage.config, storage.bucket, key; raw=true) catch return nothing end @@ -236,14 +242,14 @@ end # Examples ```julia -julia> keys = list_files(storage) +julia> keys = s3ListFiles(storage) 3-element Vector{String}: "test1.json" "test2.json" "users-1002.json" ``` """ -function list_files(storage::GarageStorage) +function s3ListFiles(storage::GarageStorage) # Use delimiter="" to get all individual keys (no grouping by "directory") objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="") return [obj["Key"] for obj in objects] @@ -259,13 +265,196 @@ end # Examples ```julia -julia> delete_file(storage, "test.json") +julia> s3DeleteFile(storage, "test.json") ``` """ -function delete_file(storage::GarageStorage, key::String) +function s3DeleteFile(storage::GarageStorage, key::String) AWSS3.s3_delete(storage.config, storage.bucket, key) end +""" + s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String + +Generate a pre-signed URL for an object in the Garage S3 bucket. + +A pre-signed URL grants temporary access without AWS credentials. Signature is computed +using AWS Signature Version 4 (HMAC-SHA256). + +# Arguments +- `storage::GarageStorage`: Storage client from `GarageStorage(endpoint, key, secret, bucket)`. +- `key::String`: Object key. Supports nested paths like `"folder/subfolder/file.txt"`. +- `seconds::Int`: URL validity in seconds. Default: 3600. + +# Keyword Arguments +- `verb::String = "GET"`: HTTP method (`"GET"` downloads, `"PUT"` uploads). +- `content_type::String = "application/octet-stream"`: Required `Content-Type` header. + For PUT, set to the file's MIME type (`"image/png"`, `"application/json"`, etc.). +- `signature_version::String = "v4"`: Always use `"v4"` (v2 is deprecated). + +# Return +`String`: The pre-signed URL. + +# Examples + +```julia +using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile + +storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "sommpanion" +) + +# Upload — PUT with curl +url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg") +# curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' + +# Download — GET with curl +url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET") +# curl → returns file content + +# Upload JSON +url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json") +# curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' + +# Delete +url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE") +# curl -X DELETE url + +# Multiple uploads (chat app) +urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300; + verb="PUT", content_type="image/png") + for _ in 1:10] + +# Signature differs by verb and content_type +url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET") +url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain") +``` + +# Notes +- URLs are valid until `seconds` expires; they can be used **multiple times** (not one-time). +- Server clocks must be within ~5 minutes of the Garage server (UTC time in signature). +- PUT uploads require the client's `Content-Type` to match `content_type`. +- For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`. +""" +function s3SignUrl( + storage::GarageStorage, + key::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + signature_version::String="v4", +)::String + config = storage.config + creds = config.credentials + bucket = storage.bucket + region = config.region + endpoint = strip(config.endpoint, '/') + scheme = "AWS4" + algorithm = "HMAC-SHA256" + terminator = "aws4_request" + service = "s3" + + now_datetime = Dates.now(Dates.UTC) + datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") + date_stamp = Dates.format(now_datetime, "YYYYmmdd") + + scope = "$date_stamp/$region/$service/$terminator" + + headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host) + if verb == "PUT" + headers["Content-Type"] = content_type + end + sort!(headers; by=name -> lowercase(name)) + canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") + + query = OrderedDict{String,String}( + "X-Amz-Expires" => string(seconds), + "X-Amz-Algorithm" => "$scheme-$algorithm", + "X-Amz-Credential" => "$(creds.access_key_id)/$scope", + "X-Amz-Date" => datetime_stamp, + "X-Amz-SignedHeaders" => canonical_header_names, + ) + if !isempty(creds.token) + query["X-Amz-Security-Token"] = creds.token + end + + canonical_headers = join( + map( + header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", + collect(headers), + ), + ) + + resource_path = "/$bucket/$key" + + # Build canonical query string: sort by key=value pair lexicographically, + # URL-encoding each key and value (matching Garage's uri_encode) + sorted_q = sort( + [URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)], + by=lowercase, + ) + canonical_query = join(sorted_q, "&") + + canonical_request = string( + "$verb\n", + resource_path, + "\n", + canonical_query, + "\n", + canonical_headers, + "\n", + canonical_header_names, + "\n", + "UNSIGNED-PAYLOAD", + ) + + string_to_sign = string( + "$scheme-$algorithm\n", + "$datetime_stamp\n", + "$scope\n", + bytes2hex(digest(MD_SHA256, canonical_request)), + ) + + key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key)) + key_date = digest(MD_SHA256, date_stamp, key_secret_bytes) + key_region = digest(MD_SHA256, region, key_date) + key_service = digest(MD_SHA256, service, key_region) + key_signing = digest(MD_SHA256, terminator, key_service) + signature = digest(MD_SHA256, string_to_sign, key_signing) + + query["X-Amz-Signature"] = bytes2hex(signature) + + query_string = URIs.escapeuri(query) + return "$(endpoint)/$bucket/$key?$query_string" +end + +""" Generate a pre-signed URL using the Garage S3 web endpoint (s3-web). + +Useful for direct browser access — URLs look like +`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...` + +Arguments and keyword arguments are the same as `s3SignUrl`. +""" +function s3SignUrlWeb( + storage::GarageStorage, + key::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + signature_version::String="v4", +)::String + web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web") + web_creds = SimpleCredentials( + storage.config.credentials.access_key_id, + storage.config.credentials.secret_key, + storage.config.credentials.token, + ) + web_config = GarageConfig(web_endpoint, storage.config.region, web_creds) + web_storage = GarageStorage(web_config, storage.bucket) + return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version) +end # =================================================================== # 3. Bucket Lifecycle Configuration @@ -323,10 +512,10 @@ end # Example ```julia rule = LifecycleExpiration("uploads/", 30) -set_lifecycle_expiration(storage, rule) +s3SetLifecycleExpiration(storage, rule) ``` """ -function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration) +function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration) xml_body = build_lifecycle_xml(rule) args = Dict{String,Any}("body" => xml_body) return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config) @@ -342,10 +531,10 @@ end # Example ```julia -config = get_lifecycle(storage) +config = s3GetLifecycle(storage) ``` """ -function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} +function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} try return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config) catch e @@ -366,10 +555,10 @@ end # Example ```julia -delete_lifecycle(storage) +s3DeleteLifecycle(storage) ``` """ -function delete_lifecycle(storage::GarageStorage) +function s3DeleteLifecycle(storage::GarageStorage) return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config) end diff --git a/test/etc.jl b/test/etc.jl index e69de29..816297d 100644 --- a/test/etc.jl +++ b/test/etc.jl @@ -0,0 +1,130 @@ +curl -X PUT -H "Content-Type: application/json" \ + -d '{"test": "data"}' \ + "https://s3-api.yiem.cc/sommpanion/pcimage.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T103745Z&X-Amz-Expires=300&X-Amz-Security-Token=&X-Amz-SignedHeaders=host&X-Amz-Signature=0e195a48f824a2f8ffcb06461396cd6a8bddd5b61d93e9006f947ae10ecf525f" + + + + + + + + + +curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ + -H 'Host: s3-api.yiem.cc' \ + -H 'Content-Type: image/jpeg' \ + "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T030315Z&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=1107a31f36f8b8ceb9663d69b9a7be7733df14d019e700e750944cbdad50df5d" + + + + + + + + + + + + + + + + +read +./garage.toml +./Caddyfile +./src/garageS3.jl + + + +julia> url = GeneralUtils.s3_sign_url(storage1, "photo.jpg", 36000; verb="PUT", content_type="image/jpeg") +"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T145004Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=92cd634dbb0f66db4b8b039ef5939c138203de260e8342fe27ef78979d90a824" + +--- + + + + + + + + + + + +ton@server12vm-dev:~/docker-apps/sommpanion/etc/AWS.jl$ +curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ + -H 'Host: s3-api.yiem.cc' \ + -H 'Content-Type: image/jpeg' \ + "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02" + +Note: Unnecessary use of -X or --request, PUT is already inferred. +* Trying 110.77.160.154:443... +* Connected to s3-api.yiem.cc (110.77.160.154) port 443 (#0) +* ALPN: offers h2,http/1.1 +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +* CAfile: /etc/ssl/certs/ca-certificates.crt +* CApath: /etc/ssl/certs +* TLSv1.3 (IN), TLS handshake, Server hello (2): +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +* TLSv1.3 (IN), TLS handshake, Certificate (11): +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +* TLSv1.3 (IN), TLS handshake, Finished (20): +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +* TLSv1.3 (OUT), TLS handshake, Finished (20): +* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256 +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=s3-api.yiem.cc +* start date: Aug 27 04:27:02 2026 GMT +* expire date: Nov 25 04:27:01 2026 GMT +* subjectAltName: host "s3-api.yiem.cc" matched cert's "s3-api.yiem.cc" +* issuer: C=US; O=Let's Encrypt; CN=YE2 +* SSL certificate verify ok. +* using HTTP/2 +* h2h3 [:method: PUT] +* h2h3 [:path: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02] +* h2h3 [:scheme: https] +* h2h3 [:authority: s3-api.yiem.cc] +* h2h3 [user-agent: curl/7.88.1] +* h2h3 [accept: */*] +* h2h3 [content-type: image/jpeg] +* h2h3 [content-length: 439490] +* Using Stream ID: 1 (easy handle 0x559a08824c80) +> PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 HTTP/2 +> Host: s3-api.yiem.cc +> user-agent: curl/7.88.1 +> accept: */* +> content-type: image/jpeg +> content-length: 439490 +> +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +* We are completely uploaded and fine +< HTTP/2 403 +< access-control-allow-origin: * +< alt-svc: h3=":443"; ma=2592000 +< content-type: application/xml +< date: Fri, 04 Sep 2026 15:23:13 GMT +< via: 1.1 Caddy +< content-length: 190 +< +* Connection #0 to host s3-api.yiem.cc left intact +AccessDeniedForbidden: Invalid signature/sommpanion/photo.jpggarage + +2026-09-04T15:23:13.796037Z INFO garage_api_common::generic_server: 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 +2026-09-04T15:23:13.796077Z DEBUG garage_api_common::generic_server: Request { method: PUT, uri: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02, version: HTTP/1.1, headers: {"host": "s3-api.yiem.cc", "user-agent": "curl/7.88.1", "content-length": "439490", "accept": "*/*", "content-type": "image/jpeg", "via": "2.0 Caddy", "x-forwarded-for": "110.77.160.154", "x-forwarded-host": "s3-api.yiem.cc", "x-forwarded-proto": "https", "accept-encoding": "gzip"}, body: Body(Streaming) } +2026-09-04T15:23:13.796181Z DEBUG garage_api_common::generic_server: Endpoint: PutObject +2026-09-04T15:23:13.796461Z INFO garage_api_common::generic_server: error 403 Forbidden, Forbidden: Invalid signature in response to 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 + + + + + + + + + + + + + diff --git a/test/runtests.jl b/test/runtests.jl index 8ef2cf7..189c795 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,9 +1,7 @@ using Test using GeneralUtils using GeneralUtils.garageS3 -using AWS -using Random: randstring -using UUIDs: uuid4 +using HTTP @testset "garageS3.jl" begin @@ -18,10 +16,10 @@ using UUIDs: uuid4 @testset "SimpleCredentials AWS extensions" begin creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1") - @test AWS.credentials(creds) === creds - @test AWS.check_credentials(creds) === creds - @test AWS.refresh!(creds; force=false) === creds - @test AWS.refresh!(creds) === creds + @test GeneralUtils.garageS3.AWS.credentials(creds) === creds + @test GeneralUtils.garageS3.AWS.check_credentials(creds) === creds + @test GeneralUtils.garageS3.AWS.refresh!(creds; force=false) === creds + @test GeneralUtils.garageS3.AWS.refresh!(creds) === creds end # --- GarageConfig --- @@ -32,57 +30,57 @@ using UUIDs: uuid4 @test config.endpoint == "https://s3-api.example.com" @test config.region == "us-east-1" @test config.credentials === creds - @test config isa AWS.AbstractAWSConfig + @test config isa GeneralUtils.garageS3.AWS.AbstractAWSConfig end @testset "GarageConfig AWS extensions" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) - @test AWS.region(config) == "garage" - @test AWS.credentials(config) === creds + @test GeneralUtils.garageS3.AWS.region(config) == "garage" + @test GeneralUtils.garageS3.AWS.credentials(config) === creds end - # --- AWS.generate_service_url --- + # --- GeneralUtils.garageS3.AWS.generate_service_url --- @testset "generate_service_url with resource starting with /" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/mybucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/mybucket/key") @test url == "https://s3-api.example.com/mybucket/key" end @testset "generate_service_url with resource NOT starting with /" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) - url = AWS.generate_service_url(config, "s3", "mybucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "mybucket/key") @test url == "https://s3-api.example.com/mybucket/key" end @testset "generate_service_url strips trailing slashes from endpoint" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "https://s3-api.example.com/bucket/key" end @testset "generate_service_url with multiple trailing slashes" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "https://s3-api.example.com/bucket/key" end @testset "generate_service_url preserves http endpoint" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "http://localhost:3900/bucket/key" end @testset "generate_service_url with http endpoint and trailing slash" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "http://localhost:3900/bucket/key" end @@ -118,17 +116,17 @@ using UUIDs: uuid4 @test storage isa GarageStorage end - # --- put_file callable struct --- + # --- s3PutFile callable struct --- - @testset "put_file callable struct type is correct" begin + @testset "s3PutFile callable struct type is correct" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") - uploader = put_file(storage) - @test uploader isa put_file + uploader = s3PutFile(storage) + @test uploader isa s3PutFile end - @testset "put_file callable struct construction stores storage" begin + @testset "s3PutFile callable struct construction stores storage" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") - uploader = put_file(storage) + uploader = s3PutFile(storage) @test uploader.storage === storage end @@ -142,11 +140,11 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-$(randstring(8)).json" + test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" - # Test put_file(storage, key, data) - successful upload - result = put_file(storage, test_key, test_data) + # Test s3PutFile(storage, key, data) - successful upload + result = s3PutFile(storage, test_key, test_data) @test result !== nothing @test haskey(result, :api) @test haskey(result, :web) @@ -154,35 +152,35 @@ using UUIDs: uuid4 @test occursin("s3-api", result.api) @test occursin("s3-web", result.web) - # Test get_file - retrieve the uploaded data - downloaded = get_file(storage, test_key) + # Test s3GetFile - retrieve the uploaded data + downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test downloaded isa Vector{UInt8} @test String(downloaded) == test_data - # Test list_files - verify key appears in listing - keys = list_files(storage) + # Test s3ListFiles - verify key appears in listing + keys = s3ListFiles(storage) @test test_key in keys - # Test put_file callable struct - test_key2 = "testrunner-callable-$(randstring(8)).json" - callable_result = put_file(storage)(test_key2, "callable-data") + # Test s3PutFile callable struct + test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json" + callable_result = s3PutFile(storage)(test_key2, "callable-data") @test callable_result !== nothing @test callable_result.api != "" @test callable_result.web != "" # Verify callable upload - downloaded2 = get_file(storage, test_key2) + downloaded2 = s3GetFile(storage, test_key2) @test String(downloaded2) == "callable-data" - # Test delete_file - delete_file(storage, test_key) - deleted_data = get_file(storage, test_key) + # Test s3DeleteFile + s3DeleteFile(storage, test_key) + deleted_data = s3GetFile(storage, test_key) @test deleted_data === nothing # Cleanup callable test object - delete_file(storage, test_key2) - deleted_data2 = get_file(storage, test_key2) + s3DeleteFile(storage, test_key2) + deleted_data2 = s3GetFile(storage, test_key2) @test deleted_data2 === nothing end @@ -194,11 +192,11 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "single/slash/test-$(randstring(8)).json" + test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"level\": 1, \"nested\": true}" # Upload nested key - result = put_file(storage, test_key, test_data) + result = s3PutFile(storage, test_key, test_data) @test result isa NamedTuple @test haskey(result, :api) @test haskey(result, :web) @@ -206,17 +204,17 @@ using UUIDs: uuid4 @test occursin("single/slash/", result.web) # Download and verify - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Verify in listing - keys = list_files(storage) + keys = s3ListFiles(storage) @test test_key in keys # Delete and verify gone - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end @testset "integration: nested key CRUD - multi level" begin @@ -227,28 +225,28 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "deep/nested/folder/subfolder/test-$(randstring(8)).json" + test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}" # Upload deeply nested key - result = put_file(storage, test_key, test_data) + result = s3PutFile(storage, test_key, test_data) @test result isa NamedTuple @test occursin("deep/nested/folder/subfolder/", result.api) @test occursin("s3-api", result.api) @test occursin("s3-web", result.web) # Download and verify - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Verify in listing - keys = list_files(storage) + keys = s3ListFiles(storage) @test test_key in keys # Delete and verify gone - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end @testset "integration: nested key with callable struct" begin @@ -259,29 +257,29 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "callable/nested/test-$(randstring(8)).json" + test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"callable\": true, \"nested\": true}" - uploader = put_file(storage) + uploader = s3PutFile(storage) result = uploader(test_key, test_data) @test result isa NamedTuple @test occursin("callable/nested/", result.api) - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test String(downloaded) == test_data - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end - @testset "integration: get_file returns nothing for missing key" begin + @testset "integration: s3GetFile returns nothing for missing key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - result = get_file(storage, "nonexistent-key-$(randstring(8)).json") + result = s3GetFile(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json") @test result === nothing end @@ -293,20 +291,20 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-binary-$(randstring(8)).bin" + test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin" binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] - result = put_file(storage, test_key, binary_data) + result = s3PutFile(storage, test_key, binary_data) @test result !== nothing - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test downloaded == binary_data - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end - @testset "integration: list_files returns vector of strings" begin + @testset "integration: s3ListFiles returns vector of strings" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", @@ -314,14 +312,14 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-list-$(randstring(8)).txt" - put_file(storage, test_key, "list-test-data") + test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt" + s3PutFile(storage, test_key, "list-test-data") - keys = list_files(storage) + keys = s3ListFiles(storage) @test keys isa Vector{String} @test test_key in keys - delete_file(storage, test_key) + s3DeleteFile(storage, test_key) end @testset "integration: url format validation" begin @@ -332,15 +330,15 @@ using UUIDs: uuid4 "testbucket" ) - test_key = "testrunner-url-$(randstring(8)).json" - result = put_file(storage, test_key, "url-test") + test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json" + result = s3PutFile(storage, test_key, "url-test") @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") @test occursin("/$test_key", result.api) @test startswith(result.web, "https://s3-web.yiem.cc/testbucket/") @test occursin("/$test_key", result.web) - delete_file(storage, test_key) + s3DeleteFile(storage, test_key) end # --- Lifecycle configuration tests --- @@ -398,23 +396,23 @@ using UUIDs: uuid4 ) # Should return nothing when no lifecycle is set - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc === nothing # Set lifecycle rule = LifecycleExpiration("lifecycle-test/", 45) - set_lifecycle_expiration(storage, rule) + s3SetLifecycleExpiration(storage, rule) # Should return configuration after setting - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc !== nothing @test haskey(lc, "Rule") # Delete lifecycle - delete_lifecycle(storage) + s3DeleteLifecycle(storage) # Should return nothing after deletion - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc === nothing end @@ -427,14 +425,14 @@ using UUIDs: uuid4 ) rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule") - set_lifecycle_expiration(storage, rule) + s3SetLifecycleExpiration(storage, rule) - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc !== nothing @test haskey(lc, "Rule") - delete_lifecycle(storage) - @test get_lifecycle(storage) === nothing + s3DeleteLifecycle(storage) + @test s3GetLifecycle(storage) === nothing end @testset "integration: lifecycle enabled/disabled toggle" begin @@ -447,17 +445,249 @@ using UUIDs: uuid4 # Set enabled rule rule1 = LifecycleExpiration("toggle/", 10; enabled=true) - set_lifecycle_expiration(storage, rule1) - lc = get_lifecycle(storage) + s3SetLifecycleExpiration(storage, rule1) + lc = s3GetLifecycle(storage) @test lc !== nothing # Override with disabled rule rule2 = LifecycleExpiration("toggle/", 10; enabled=false) - set_lifecycle_expiration(storage, rule2) - lc = get_lifecycle(storage) + s3SetLifecycleExpiration(storage, rule2) + lc = s3GetLifecycle(storage) @test lc !== nothing - delete_lifecycle(storage) + s3DeleteLifecycle(storage) + end + + # --- Pre-signed URL tests --- + + @testset "s3SignUrl returns a string" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3SignUrl(storage, "test.json", 3600) + @test url isa String + @test !isempty(url) + end + + @testset "s3SignUrl URL format contains correct endpoint, bucket, key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3SignUrl(storage, "mykey.json", 1800) + @test occursin("https://s3-api.yiem.cc", url) + @test occursin("/testbucket/mykey.json?", url) + end + + @testset "s3SignUrl contains required query parameters" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3SignUrl(storage, "test.json", 3600) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url) + @test occursin("X-Amz-Expires=3600", url) + @test occursin("X-Amz-Credential=", url) + @test occursin("X-Amz-Date=", url) + @test occursin("X-Amz-SignedHeaders=", url) + @test occursin("X-Amz-Signature=", url) + end + + @testset "s3SignUrl with custom expiry duration" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3SignUrl(storage, "test.json", 600) + @test occursin("X-Amz-Expires=600", url) + end + + @testset "s3SignUrl with GET verb produces different signature than default" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url_get = s3SignUrl(storage, "test.json", 3600; verb="GET") + url_default = s3SignUrl(storage, "test.json", 3600) + @test occursin("X-Amz-Signature=", url_get) + @test occursin("X-Amz-Signature=", url_default) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get) + end + + @testset "s3SignUrl with PUT verb and content_type" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url_put = s3SignUrl(storage, "upload.json", 300; verb="PUT", content_type="application/json") + url_get = s3SignUrl(storage, "upload.json", 300; verb="GET") + @test occursin("X-Amz-Signature=", url_put) + @test occursin("X-Amz-Signature=", url_get) + end + + @testset "s3SignUrl with special characters in key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + test_key = "folder/sub folder/file with spaces.json" + url = s3SignUrl(storage, test_key, 3600) + @test occursin("testbucket/", url) + end + + @testset "s3SignUrl with deeply nested key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + test_key = "a/b/c/d/e/deep-file.json" + url = s3SignUrl(storage, test_key, 3600) + @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url) + end + + @testset "integration: pre-signed URL can download existing file" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}" + s3PutFile(storage, test_key, test_data) + + url = s3SignUrl(storage, test_key, 300) + @test occursin("https://s3-api.yiem.cc/testbucket/", url) + + s3DeleteFile(storage, test_key) + end + + # --- s3SignUrl end-to-end: PUT upload, GET download, DELETE --- + + @testset "integration: s3SignUrl PUT upload via pre-signed URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-signurl-put-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"signed\": true, \"upload\": \"put\"}" + content_type = "application/json" + + # Generate pre-signed URL for PUT + put_url = s3SignUrl(storage, test_key, 300; verb="PUT", content_type=content_type) + @test occursin("X-Amz-Signature=", put_url) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", put_url) + @test occursin("X-Amz-SignedHeaders=", put_url) + + # Upload via HTTP request using the signed URL + response = HTTP.request( + "PUT", + put_url, + ["Content-Type" => content_type], + test_data + ) + @test response.status == 200 + + # Verify upload via s3GetFile + downloaded = s3GetFile(storage, test_key) + @test downloaded !== nothing + @test String(downloaded) == test_data + + # Cleanup + s3DeleteFile(storage, test_key) + end + + @testset "integration: s3SignUrl GET download via pre-signed URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-signurl-get-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"signed\": true, \"download\": \"get\"}" + + # Upload via s3PutFile first + s3PutFile(storage, test_key, test_data) + + # Generate pre-signed URL for GET + get_url = s3SignUrl(storage, test_key, 300; verb="GET") + @test occursin("X-Amz-Signature=", get_url) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", get_url) + + # Download via HTTP GET request using the signed URL + response = HTTP.request("GET", get_url) + @test response.status == 200 + @test String(response.body) == test_data + + # Cleanup + s3DeleteFile(storage, test_key) + end + + @testset "integration: s3SignUrl DELETE via pre-signed URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-signurl-delete-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"signed\": true, \"delete\": true}" + + # Upload first + s3PutFile(storage, test_key, test_data) + + # Verify it exists + @test s3GetFile(storage, test_key) !== nothing + + # Generate pre-signed URL for DELETE + delete_url = s3SignUrl(storage, test_key, 300; verb="DELETE") + @test occursin("X-Amz-Signature=", delete_url) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", delete_url) + + # Delete via HTTP DELETE request using the signed URL + response = HTTP.request("DELETE", delete_url) + @test response.status == 204 + + # Verify deletion + @test s3GetFile(storage, test_key) === nothing + end + + @testset "integration: s3SignUrlWeb generates web endpoint URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3SignUrlWeb(storage, "web-test.json", 3600) + @test url isa String + @test occursin("https://s3-web.yiem.cc", url) + @test occursin("/testbucket/web-test.json?", url) + @test occursin("X-Amz-Signature=", url) end end diff --git a/test/winebottle.jpg b/test/winebottle.jpg new file mode 100644 index 0000000..7de5a42 Binary files /dev/null and b/test/winebottle.jpg differ