using Test using GeneralUtils using GeneralUtils.garageS3 using HTTP @testset "garageS3.jl" begin # --- SimpleCredentials --- @testset "SimpleCredentials construction" begin creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken") @test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE" @test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" @test creds.token == "sessiontoken" end @testset "SimpleCredentials AWS extensions" begin creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1") @test GeneralUtils.garageS3.AWS.credentials(creds) === creds @test GeneralUtils.garageS3.AWS.check_credentials(creds) === creds @test GeneralUtils.garageS3.AWS.refresh!(creds; force=false) === creds @test GeneralUtils.garageS3.AWS.refresh!(creds) === creds end # --- GarageConfig --- @testset "GarageConfig construction" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds) @test config.endpoint == "https://s3-api.example.com" @test config.region == "us-east-1" @test config.credentials === creds @test config isa GeneralUtils.garageS3.AWS.AbstractAWSConfig end @testset "GarageConfig AWS extensions" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) @test GeneralUtils.garageS3.AWS.region(config) == "garage" @test GeneralUtils.garageS3.AWS.credentials(config) === creds end # --- GeneralUtils.garageS3.AWS.generate_service_url --- @testset "generate_service_url with resource starting with /" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/mybucket/key") @test url == "https://s3-api.example.com/mybucket/key" end @testset "generate_service_url with resource NOT starting with /" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "mybucket/key") @test url == "https://s3-api.example.com/mybucket/key" end @testset "generate_service_url strips trailing slashes from endpoint" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds) url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "https://s3-api.example.com/bucket/key" end @testset "generate_service_url with multiple trailing slashes" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds) url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "https://s3-api.example.com/bucket/key" end @testset "generate_service_url preserves http endpoint" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds) url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "http://localhost:3900/bucket/key" end @testset "generate_service_url with http endpoint and trailing slash" begin creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds) url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key") @test url == "http://localhost:3900/bucket/key" end # --- GarageStorage --- @testset "GarageStorage default region" begin storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket") @test storage.bucket == "mybucket" @test storage.config.region == "garage" @test storage.config.endpoint == "https://s3-api.example.com" end @testset "GarageStorage custom region" begin storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region") @test storage.config.region == "custom-region" end @testset "GarageStorage endpoint without https" begin storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket") @test storage.config.endpoint == "http://localhost:3900" end @testset "GarageStorage credentials are SimpleCredentials" begin storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket") @test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials @test storage.config.credentials.access_key_id == "mykey" @test storage.config.credentials.secret_key == "mysecret" @test storage.config.credentials.token == "" end @testset "GarageStorage returns correct struct type" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") @test storage isa GarageStorage end # --- s3PutFile callable struct --- @testset "s3PutFile callable struct type is correct" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") uploader = s3PutFile(storage) @test uploader isa s3PutFile end @testset "s3PutFile callable struct construction stores storage" begin storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") uploader = s3PutFile(storage) @test uploader.storage === storage end # --- Integration tests with real Garage server --- @testset "integration: full CRUD lifecycle" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" # Test s3PutFile(storage, key, data) - successful upload result = s3PutFile(storage, test_key, test_data) @test result !== nothing @test haskey(result, :api) @test haskey(result, :web) @test occursin("testbucket", result.api) @test occursin("s3-api", result.api) @test occursin("s3-web", result.web) # Test s3GetFile - retrieve the uploaded data downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test downloaded isa Vector{UInt8} @test String(downloaded) == test_data # Test s3ListFiles - verify key appears in listing keys = s3ListFiles(storage) @test test_key in keys # Test s3PutFile callable struct test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json" callable_result = s3PutFile(storage)(test_key2, "callable-data") @test callable_result !== nothing @test callable_result.api != "" @test callable_result.web != "" # Verify callable upload downloaded2 = s3GetFile(storage, test_key2) @test String(downloaded2) == "callable-data" # Test s3DeleteFile s3DeleteFile(storage, test_key) deleted_data = s3GetFile(storage, test_key) @test deleted_data === nothing # Cleanup callable test object s3DeleteFile(storage, test_key2) deleted_data2 = s3GetFile(storage, test_key2) @test deleted_data2 === nothing end @testset "integration: nested key CRUD - single level" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"level\": 1, \"nested\": true}" # Upload nested key result = s3PutFile(storage, test_key, test_data) @test result isa NamedTuple @test haskey(result, :api) @test haskey(result, :web) @test occursin("single/slash/", result.api) @test occursin("single/slash/", result.web) # Download and verify downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Verify in listing keys = s3ListFiles(storage) @test test_key in keys # Delete and verify gone s3DeleteFile(storage, test_key) @test s3GetFile(storage, test_key) === nothing end @testset "integration: nested key CRUD - multi level" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}" # Upload deeply nested key result = s3PutFile(storage, test_key, test_data) @test result isa NamedTuple @test occursin("deep/nested/folder/subfolder/", result.api) @test occursin("s3-api", result.api) @test occursin("s3-web", result.web) # Download and verify downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Verify in listing keys = s3ListFiles(storage) @test test_key in keys # Delete and verify gone s3DeleteFile(storage, test_key) @test s3GetFile(storage, test_key) === nothing end @testset "integration: nested key with callable struct" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"callable\": true, \"nested\": true}" uploader = s3PutFile(storage) result = uploader(test_key, test_data) @test result isa NamedTuple @test occursin("callable/nested/", result.api) downloaded = s3GetFile(storage, test_key) @test String(downloaded) == test_data s3DeleteFile(storage, test_key) @test s3GetFile(storage, test_key) === nothing end @testset "integration: s3GetFile returns nothing for missing key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) result = s3GetFile(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json") @test result === nothing end @testset "integration: binary data upload and download" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin" binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] result = s3PutFile(storage, test_key, binary_data) @test result !== nothing downloaded = s3GetFile(storage, test_key) @test downloaded == binary_data s3DeleteFile(storage, test_key) @test s3GetFile(storage, test_key) === nothing end @testset "integration: s3ListFiles returns vector of strings" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt" s3PutFile(storage, test_key, "list-test-data") keys = s3ListFiles(storage) @test keys isa Vector{String} @test test_key in keys s3DeleteFile(storage, test_key) end @testset "integration: url format validation" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json" result = s3PutFile(storage, test_key, "url-test") @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") @test occursin("/$test_key", result.api) @test startswith(result.web, "https://s3-web.yiem.cc/testbucket/") @test occursin("/$test_key", result.web) s3DeleteFile(storage, test_key) end # --- Lifecycle configuration tests --- @testset "LifecycleExpiration construction with default ID" begin rule = LifecycleExpiration("uploads/", 30) @test rule.prefix == "uploads/" @test rule.days == 30 @test rule.enabled == true @test rule.id == "expire-uploads_-30d" end @testset "LifecycleExpiration construction with custom ID" begin rule = LifecycleExpiration("data/", 7; id="my-rule") @test rule.prefix == "data/" @test rule.days == 7 @test rule.enabled == true @test rule.id == "my-rule" end @testset "LifecycleExpiration with disabled flag" begin rule = LifecycleExpiration("temp/", 1; enabled=false) @test rule.enabled == false end @testset "LifecycleExpiration with nested prefix" begin rule = LifecycleExpiration("deep/nested/", 90) @test rule.prefix == "deep/nested/" end @testset "build_lifecycle_xml generates valid XML" begin rule = LifecycleExpiration("test/", 14; id="test-rule") xml = build_lifecycle_xml(rule) @test occursin("", xml) @test occursin("test-rule", xml) @test occursin("test/", xml) @test occursin("Enabled", xml) @test occursin("14", xml) @test occursin("", xml) end @testset "build_lifecycle_xml generates disabled rule XML" begin rule = LifecycleExpiration("temp/", 1; enabled=false) xml = build_lifecycle_xml(rule) @test occursin("Disabled", xml) end @testset "integration: lifecycle CRUD - set and get" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) # Should return nothing when no lifecycle is set lc = s3GetLifecycle(storage) @test lc === nothing # Set lifecycle rule = LifecycleExpiration("lifecycle-test/", 45) s3SetLifecycleExpiration(storage, rule) # Should return configuration after setting lc = s3GetLifecycle(storage) @test lc !== nothing @test haskey(lc, "Rule") # Delete lifecycle s3DeleteLifecycle(storage) # Should return nothing after deletion lc = s3GetLifecycle(storage) @test lc === nothing end @testset "integration: lifecycle with nested prefix" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule") s3SetLifecycleExpiration(storage, rule) lc = s3GetLifecycle(storage) @test lc !== nothing @test haskey(lc, "Rule") s3DeleteLifecycle(storage) @test s3GetLifecycle(storage) === nothing end @testset "integration: lifecycle enabled/disabled toggle" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) # Set enabled rule rule1 = LifecycleExpiration("toggle/", 10; enabled=true) s3SetLifecycleExpiration(storage, rule1) lc = s3GetLifecycle(storage) @test lc !== nothing # Override with disabled rule rule2 = LifecycleExpiration("toggle/", 10; enabled=false) s3SetLifecycleExpiration(storage, rule2) lc = s3GetLifecycle(storage) @test lc !== nothing s3DeleteLifecycle(storage) end # --- Pre-signed URL tests --- @testset "s3SignUrl returns a string" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url = s3SignUrl(storage, "test.json", 3600) @test url isa String @test !isempty(url) end @testset "s3SignUrl URL format contains correct endpoint, bucket, key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url = s3SignUrl(storage, "mykey.json", 1800) @test occursin("https://s3-api.yiem.cc", url) @test occursin("/testbucket/mykey.json?", url) end @testset "s3SignUrl contains required query parameters" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url = s3SignUrl(storage, "test.json", 3600) @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url) @test occursin("X-Amz-Expires=3600", url) @test occursin("X-Amz-Credential=", url) @test occursin("X-Amz-Date=", url) @test occursin("X-Amz-SignedHeaders=", url) @test occursin("X-Amz-Signature=", url) end @testset "s3SignUrl with custom expiry duration" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url = s3SignUrl(storage, "test.json", 600) @test occursin("X-Amz-Expires=600", url) end @testset "s3SignUrl with GET verb produces different signature than default" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url_get = s3SignUrl(storage, "test.json", 3600; verb="GET") url_default = s3SignUrl(storage, "test.json", 3600) @test occursin("X-Amz-Signature=", url_get) @test occursin("X-Amz-Signature=", url_default) @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get) end @testset "s3SignUrl with PUT verb and content_type" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url_put = s3SignUrl(storage, "upload.json", 300; verb="PUT", content_type="application/json") url_get = s3SignUrl(storage, "upload.json", 300; verb="GET") @test occursin("X-Amz-Signature=", url_put) @test occursin("X-Amz-Signature=", url_get) end @testset "s3SignUrl with special characters in key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "folder/sub folder/file with spaces.json" url = s3SignUrl(storage, test_key, 3600) @test occursin("testbucket/", url) end @testset "s3SignUrl with deeply nested key" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "a/b/c/d/e/deep-file.json" url = s3SignUrl(storage, test_key, 3600) @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url) end @testset "integration: pre-signed URL can download existing file" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}" s3PutFile(storage, test_key, test_data) url = s3SignUrl(storage, test_key, 300) @test occursin("https://s3-api.yiem.cc/testbucket/", url) s3DeleteFile(storage, test_key) end # --- s3SignUrl end-to-end: PUT upload, GET download, DELETE --- @testset "integration: s3SignUrl PUT upload via pre-signed URL" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-signurl-put-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"signed\": true, \"upload\": \"put\"}" content_type = "application/json" # Generate pre-signed URL for PUT put_url = s3SignUrl(storage, test_key, 300; verb="PUT", content_type=content_type) @test occursin("X-Amz-Signature=", put_url) @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", put_url) @test occursin("X-Amz-SignedHeaders=", put_url) # Upload via HTTP request using the signed URL response = HTTP.request( "PUT", put_url, ["Content-Type" => content_type], test_data ) @test response.status == 200 # Verify upload via s3GetFile downloaded = s3GetFile(storage, test_key) @test downloaded !== nothing @test String(downloaded) == test_data # Cleanup s3DeleteFile(storage, test_key) end @testset "integration: s3SignUrl GET download via pre-signed URL" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-signurl-get-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"signed\": true, \"download\": \"get\"}" # Upload via s3PutFile first s3PutFile(storage, test_key, test_data) # Generate pre-signed URL for GET get_url = s3SignUrl(storage, test_key, 300; verb="GET") @test occursin("X-Amz-Signature=", get_url) @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", get_url) # Download via HTTP GET request using the signed URL response = HTTP.request("GET", get_url) @test response.status == 200 @test String(response.body) == test_data # Cleanup s3DeleteFile(storage, test_key) end @testset "integration: s3SignUrl DELETE via pre-signed URL" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) test_key = "testrunner-signurl-delete-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"signed\": true, \"delete\": true}" # Upload first s3PutFile(storage, test_key, test_data) # Verify it exists @test s3GetFile(storage, test_key) !== nothing # Generate pre-signed URL for DELETE delete_url = s3SignUrl(storage, test_key, 300; verb="DELETE") @test occursin("X-Amz-Signature=", delete_url) @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", delete_url) # Delete via HTTP DELETE request using the signed URL response = HTTP.request("DELETE", delete_url) @test response.status == 204 # Verify deletion @test s3GetFile(storage, test_key) === nothing end @testset "integration: s3SignUrlWeb generates web endpoint URL" begin storage = GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) url = s3SignUrlWeb(storage, "web-test.json", 3600) @test url isa String @test occursin("https://s3-web.yiem.cc", url) @test occursin("/testbucket/web-test.json?", url) @test occursin("X-Amz-Signature=", url) end end