diff --git a/src/garageS3.jl b/src/garageS3.jl index 29b5787..f204261 100644 --- a/src/garageS3.jl +++ b/src/garageS3.jl @@ -225,7 +225,7 @@ julia> data = String(s3GetFile(storage, "test.json")) """ function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} try - return AWSS3.s3_get(storage.config, storage.bucket, key) + return AWSS3.s3_get(storage.config, storage.bucket, key; raw=true) catch return nothing end diff --git a/test/runtests.jl b/test/runtests.jl index e19cd29..189c795 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,8 +1,7 @@ using Test using GeneralUtils using GeneralUtils.garageS3 -using AWS -using URIs +using HTTP @testset "garageS3.jl" begin @@ -17,10 +16,10 @@ using URIs @testset "SimpleCredentials AWS extensions" begin creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1") - @test AWS.credentials(creds) === creds - @test AWS.check_credentials(creds) === creds - @test AWS.refresh!(creds; force=false) === creds - @test AWS.refresh!(creds) === creds + @test GeneralUtils.garageS3.AWS.credentials(creds) === creds + @test GeneralUtils.garageS3.AWS.check_credentials(creds) === creds + @test GeneralUtils.garageS3.AWS.refresh!(creds; force=false) === creds + @test GeneralUtils.garageS3.AWS.refresh!(creds) === creds end # --- GarageConfig --- @@ -31,57 +30,57 @@ using URIs @test config.endpoint == "https://s3-api.example.com" @test config.region == "us-east-1" @test config.credentials === creds - @test config isa AWS.AbstractAWSConfig + @test config isa GeneralUtils.garageS3.AWS.AbstractAWSConfig end @testset "GarageConfig AWS extensions" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) - @test AWS.region(config) == "garage" - @test AWS.credentials(config) === creds + @test GeneralUtils.garageS3.AWS.region(config) == "garage" + @test GeneralUtils.garageS3.AWS.credentials(config) === creds end - # --- AWS.generate_service_url --- + # --- GeneralUtils.garageS3.AWS.generate_service_url --- @testset "generate_service_url with resource starting with /" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/mybucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/mybucket/key") @test url == "https://s3-api.example.com/mybucket/key" end @testset "generate_service_url with resource NOT starting with /" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) - url = AWS.generate_service_url(config, "s3", "mybucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "mybucket/key") @test url == "https://s3-api.example.com/mybucket/key" end @testset "generate_service_url strips trailing slashes from endpoint" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "https://s3-api.example.com/bucket/key" end @testset "generate_service_url with multiple trailing slashes" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "https://s3-api.example.com/bucket/key" end @testset "generate_service_url preserves http endpoint" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "http://localhost:3900/bucket/key" end @testset "generate_service_url with http endpoint and trailing slash" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds) - url = AWS.generate_service_url(config, "s3", "/bucket/key") + url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "http://localhost:3900/bucket/key" end @@ -117,17 +116,17 @@ using URIs @test storage isa GarageStorage end - # --- put_file callable struct --- + # --- s3PutFile callable struct --- - @testset "put_file callable struct type is correct" begin + @testset "s3PutFile callable struct type is correct" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") - uploader = put_file(storage) - @test uploader isa put_file + uploader = s3PutFile(storage) + @test uploader isa s3PutFile end - @testset "put_file callable struct construction stores storage" begin + @testset "s3PutFile callable struct construction stores storage" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") - uploader = put_file(storage) + uploader = s3PutFile(storage) @test uploader.storage === storage end @@ -144,8 +143,8 @@ using URIs test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" - # Test put_file(storage, key, data) - successful upload - result = put_file(storage, test_key, test_data) + # Test s3PutFile(storage, key, data) - successful upload + result = s3PutFile(storage, test_key, test_data) @test result !== nothing @test haskey(result, :api) @test haskey(result, :web) @@ -153,35 +152,35 @@ using URIs @test occursin("s3-api", result.api) @test occursin("s3-web", result.web) - # Test get_file - retrieve the uploaded data - downloaded = get_file(storage, test_key) + # Test s3GetFile - retrieve the uploaded data + downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test downloaded isa Vector{UInt8} @test String(downloaded) == test_data - # Test list_files - verify key appears in listing - keys = list_files(storage) + # Test s3ListFiles - verify key appears in listing + keys = s3ListFiles(storage) @test test_key in keys - # Test put_file callable struct + # Test s3PutFile callable struct test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json" - callable_result = put_file(storage)(test_key2, "callable-data") + callable_result = s3PutFile(storage)(test_key2, "callable-data") @test callable_result !== nothing @test callable_result.api != "" @test callable_result.web != "" # Verify callable upload - downloaded2 = get_file(storage, test_key2) + downloaded2 = s3GetFile(storage, test_key2) @test String(downloaded2) == "callable-data" - # Test delete_file - delete_file(storage, test_key) - deleted_data = get_file(storage, test_key) + # Test s3DeleteFile + s3DeleteFile(storage, test_key) + deleted_data = s3GetFile(storage, test_key) @test deleted_data === nothing # Cleanup callable test object - delete_file(storage, test_key2) - deleted_data2 = get_file(storage, test_key2) + s3DeleteFile(storage, test_key2) + deleted_data2 = s3GetFile(storage, test_key2) @test deleted_data2 === nothing end @@ -197,7 +196,7 @@ using URIs test_data = "{\"level\": 1, \"nested\": true}" # Upload nested key - result = put_file(storage, test_key, test_data) + result = s3PutFile(storage, test_key, test_data) @test result isa NamedTuple @test haskey(result, :api) @test haskey(result, :web) @@ -205,17 +204,17 @@ using URIs @test occursin("single/slash/", result.web) # Download and verify - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Verify in listing - keys = list_files(storage) + keys = s3ListFiles(storage) @test test_key in keys # Delete and verify gone - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end @testset "integration: nested key CRUD - multi level" begin @@ -230,24 +229,24 @@ using URIs test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}" # Upload deeply nested key - result = put_file(storage, test_key, test_data) + result = s3PutFile(storage, test_key, test_data) @test result isa NamedTuple @test occursin("deep/nested/folder/subfolder/", result.api) @test occursin("s3-api", result.api) @test occursin("s3-web", result.web) # Download and verify - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Verify in listing - keys = list_files(storage) + keys = s3ListFiles(storage) @test test_key in keys # Delete and verify gone - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end @testset "integration: nested key with callable struct" begin @@ -261,26 +260,26 @@ using URIs test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"callable\": true, \"nested\": true}" - uploader = put_file(storage) + uploader = s3PutFile(storage) result = uploader(test_key, test_data) @test result isa NamedTuple @test occursin("callable/nested/", result.api) - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test String(downloaded) == test_data - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end - @testset "integration: get_file returns nothing for missing key" begin + @testset "integration: s3GetFile returns nothing for missing key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - result = get_file(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json") + result = s3GetFile(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json") @test result === nothing end @@ -295,17 +294,17 @@ using URIs test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin" binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] - result = put_file(storage, test_key, binary_data) + result = s3PutFile(storage, test_key, binary_data) @test result !== nothing - downloaded = get_file(storage, test_key) + downloaded = s3GetFile(storage, test_key) @test downloaded == binary_data - delete_file(storage, test_key) - @test get_file(storage, test_key) === nothing + s3DeleteFile(storage, test_key) + @test s3GetFile(storage, test_key) === nothing end - @testset "integration: list_files returns vector of strings" begin + @testset "integration: s3ListFiles returns vector of strings" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", @@ -314,13 +313,13 @@ using URIs ) test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt" - put_file(storage, test_key, "list-test-data") + s3PutFile(storage, test_key, "list-test-data") - keys = list_files(storage) + keys = s3ListFiles(storage) @test keys isa Vector{String} @test test_key in keys - delete_file(storage, test_key) + s3DeleteFile(storage, test_key) end @testset "integration: url format validation" begin @@ -332,14 +331,14 @@ using URIs ) test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json" - result = put_file(storage, test_key, "url-test") + result = s3PutFile(storage, test_key, "url-test") @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") @test occursin("/$test_key", result.api) @test startswith(result.web, "https://s3-web.yiem.cc/testbucket/") @test occursin("/$test_key", result.web) - delete_file(storage, test_key) + s3DeleteFile(storage, test_key) end # --- Lifecycle configuration tests --- @@ -397,23 +396,23 @@ using URIs ) # Should return nothing when no lifecycle is set - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc === nothing # Set lifecycle rule = LifecycleExpiration("lifecycle-test/", 45) - set_lifecycle_expiration(storage, rule) + s3SetLifecycleExpiration(storage, rule) # Should return configuration after setting - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc !== nothing @test haskey(lc, "Rule") # Delete lifecycle - delete_lifecycle(storage) + s3DeleteLifecycle(storage) # Should return nothing after deletion - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc === nothing end @@ -426,14 +425,14 @@ using URIs ) rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule") - set_lifecycle_expiration(storage, rule) + s3SetLifecycleExpiration(storage, rule) - lc = get_lifecycle(storage) + lc = s3GetLifecycle(storage) @test lc !== nothing @test haskey(lc, "Rule") - delete_lifecycle(storage) - @test get_lifecycle(storage) === nothing + s3DeleteLifecycle(storage) + @test s3GetLifecycle(storage) === nothing end @testset "integration: lifecycle enabled/disabled toggle" begin @@ -446,53 +445,53 @@ using URIs # Set enabled rule rule1 = LifecycleExpiration("toggle/", 10; enabled=true) - set_lifecycle_expiration(storage, rule1) - lc = get_lifecycle(storage) + s3SetLifecycleExpiration(storage, rule1) + lc = s3GetLifecycle(storage) @test lc !== nothing # Override with disabled rule rule2 = LifecycleExpiration("toggle/", 10; enabled=false) - set_lifecycle_expiration(storage, rule2) - lc = get_lifecycle(storage) + s3SetLifecycleExpiration(storage, rule2) + lc = s3GetLifecycle(storage) @test lc !== nothing - delete_lifecycle(storage) + s3DeleteLifecycle(storage) end # --- Pre-signed URL tests --- - @testset "s3_sign_url returns a string" begin + @testset "s3SignUrl returns a string" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - url = s3_sign_url(storage, "test.json", 3600) + url = s3SignUrl(storage, "test.json", 3600) @test url isa String @test !isempty(url) end - @testset "s3_sign_url URL format contains correct endpoint, bucket, key" begin + @testset "s3SignUrl URL format contains correct endpoint, bucket, key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - url = s3_sign_url(storage, "mykey.json", 1800) + url = s3SignUrl(storage, "mykey.json", 1800) @test occursin("https://s3-api.yiem.cc", url) @test occursin("/testbucket/mykey.json?", url) end - @testset "s3_sign_url contains required query parameters" begin + @testset "s3SignUrl contains required query parameters" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - url = s3_sign_url(storage, "test.json", 3600) + url = s3SignUrl(storage, "test.json", 3600) @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url) @test occursin("X-Amz-Expires=3600", url) @test occursin("X-Amz-Credential=", url) @@ -501,48 +500,45 @@ using URIs @test occursin("X-Amz-Signature=", url) end - @testset "s3_sign_url with custom expiry duration" begin + @testset "s3SignUrl with custom expiry duration" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - url = s3_sign_url(storage, "test.json", 600) + url = s3SignUrl(storage, "test.json", 600) @test occursin("X-Amz-Expires=600", url) end - @testset "s3_sign_url with GET verb produces different signature than default" begin + @testset "s3SignUrl with GET verb produces different signature than default" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - url_get = s3_sign_url(storage, "test.json", 3600; verb="GET") - url_default = s3_sign_url(storage, "test.json", 3600) - # GET verb should produce a valid signature + url_get = s3SignUrl(storage, "test.json", 3600; verb="GET") + url_default = s3SignUrl(storage, "test.json", 3600) @test occursin("X-Amz-Signature=", url_get) @test occursin("X-Amz-Signature=", url_default) - # Both URLs should have the same structure but potentially different signatures @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get) end - @testset "s3_sign_url with PUT verb and content_type" begin + @testset "s3SignUrl with PUT verb and content_type" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - url_put = s3_sign_url(storage, "upload.json", 300; verb="PUT", content_type="application/json") - url_get = s3_sign_url(storage, "upload.json", 300; verb="GET") - # PUT and GET should produce different signatures (different canonical request) + url_put = s3SignUrl(storage, "upload.json", 300; verb="PUT", content_type="application/json") + url_get = s3SignUrl(storage, "upload.json", 300; verb="GET") @test occursin("X-Amz-Signature=", url_put) @test occursin("X-Amz-Signature=", url_get) end - @testset "s3_sign_url with special characters in key" begin + @testset "s3SignUrl with special characters in key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", @@ -550,11 +546,11 @@ using URIs "testbucket" ) test_key = "folder/sub folder/file with spaces.json" - url = s3_sign_url(storage, test_key, 3600) + url = s3SignUrl(storage, test_key, 3600) @test occursin("testbucket/", url) end - @testset "s3_sign_url with deeply nested key" begin + @testset "s3SignUrl with deeply nested key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", @@ -562,7 +558,7 @@ using URIs "testbucket" ) test_key = "a/b/c/d/e/deep-file.json" - url = s3_sign_url(storage, test_key, 3600) + url = s3SignUrl(storage, test_key, 3600) @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url) end @@ -576,12 +572,122 @@ using URIs test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}" - put_file(storage, test_key, test_data) + s3PutFile(storage, test_key, test_data) - url = s3_sign_url(storage, test_key, 300) + url = s3SignUrl(storage, test_key, 300) @test occursin("https://s3-api.yiem.cc/testbucket/", url) - delete_file(storage, test_key) + s3DeleteFile(storage, test_key) + end + + # --- s3SignUrl end-to-end: PUT upload, GET download, DELETE --- + + @testset "integration: s3SignUrl PUT upload via pre-signed URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-signurl-put-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"signed\": true, \"upload\": \"put\"}" + content_type = "application/json" + + # Generate pre-signed URL for PUT + put_url = s3SignUrl(storage, test_key, 300; verb="PUT", content_type=content_type) + @test occursin("X-Amz-Signature=", put_url) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", put_url) + @test occursin("X-Amz-SignedHeaders=", put_url) + + # Upload via HTTP request using the signed URL + response = HTTP.request( + "PUT", + put_url, + ["Content-Type" => content_type], + test_data + ) + @test response.status == 200 + + # Verify upload via s3GetFile + downloaded = s3GetFile(storage, test_key) + @test downloaded !== nothing + @test String(downloaded) == test_data + + # Cleanup + s3DeleteFile(storage, test_key) + end + + @testset "integration: s3SignUrl GET download via pre-signed URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-signurl-get-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"signed\": true, \"download\": \"get\"}" + + # Upload via s3PutFile first + s3PutFile(storage, test_key, test_data) + + # Generate pre-signed URL for GET + get_url = s3SignUrl(storage, test_key, 300; verb="GET") + @test occursin("X-Amz-Signature=", get_url) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", get_url) + + # Download via HTTP GET request using the signed URL + response = HTTP.request("GET", get_url) + @test response.status == 200 + @test String(response.body) == test_data + + # Cleanup + s3DeleteFile(storage, test_key) + end + + @testset "integration: s3SignUrl DELETE via pre-signed URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-signurl-delete-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"signed\": true, \"delete\": true}" + + # Upload first + s3PutFile(storage, test_key, test_data) + + # Verify it exists + @test s3GetFile(storage, test_key) !== nothing + + # Generate pre-signed URL for DELETE + delete_url = s3SignUrl(storage, test_key, 300; verb="DELETE") + @test occursin("X-Amz-Signature=", delete_url) + @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", delete_url) + + # Delete via HTTP DELETE request using the signed URL + response = HTTP.request("DELETE", delete_url) + @test response.status == 204 + + # Verify deletion + @test s3GetFile(storage, test_key) === nothing + end + + @testset "integration: s3SignUrlWeb generates web endpoint URL" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + url = s3SignUrlWeb(storage, "web-test.json", 3600) + @test url isa String + @test occursin("https://s3-web.yiem.cc", url) + @test occursin("/testbucket/web-test.json?", url) + @test occursin("X-Amz-Signature=", url) end end