diff --git a/src/garageS3.jl b/src/garageS3.jl index 7086955..a99484d 100644 --- a/src/garageS3.jl +++ b/src/garageS3.jl @@ -13,11 +13,11 @@ using AWSS3 # this file use local AWS config """ Example -storage = GeneralUtils.GarageStorage( +storage1 = GeneralUtils.GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "sommpanion-s3" + "testbucket" ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) @@ -30,7 +30,7 @@ println("Bucket keys: ", key) # test with curl curl -v \ - -H 'Host: sommpanion-s3.s3-web.yiem.cc' \ + -H 'Host: testbucket.s3-web.yiem.cc' \ http://192.168.88.106:3902/users-1002.json # test with API @@ -64,23 +64,6 @@ AWS.check_credentials(c::SimpleCredentials) = c AWS.region(aws::GarageConfig) = aws.region AWS.credentials(aws::GarageConfig) = aws.credentials -""" Generate the base URL for a Garage S3 service request. - -# Arguments -- `aws::GarageConfig` - The Garage configuration containing the endpoint. -- `service::String` - The AWS service name (e.g., `"s3"`). -- `region::String` - The AWS region string. - -# Return -- `String`: The stripped endpoint URL (trailing slashes removed). -""" -function AWS.generate_service_url(aws::GarageConfig, service::String, region::String) - return strip(aws.endpoint, '/') -end - """ Generate the full service URL including the resource path for a Garage S3 request. # Arguments @@ -183,7 +166,7 @@ function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}} println("Successfully uploaded: ", key) # object url - api = "$(storage.config.endpoint)/$(storage.bucket)/$key" + api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key" # file download URL using browser web = replace(api, "s3-api" => "s3-web") diff --git a/test/runtests.jl b/test/runtests.jl index f415525..0ce41a2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,39 +1,302 @@ using Test -using GeneralUtils: detect_keyword +using GeneralUtils +using GeneralUtils.garageS3 +using AWS +using Random: randstring +using UUIDs: uuid4 + +@testset "garageS3.jl" begin + + # --- SimpleCredentials --- + + @testset "SimpleCredentials construction" begin + creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken") + @test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE" + @test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + @test creds.token == "sessiontoken" + end + + @testset "SimpleCredentials AWS extensions" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1") + @test AWS.credentials(creds) === creds + @test AWS.check_credentials(creds) === creds + @test AWS.refresh!(creds; force=false) === creds + @test AWS.refresh!(creds) === creds + end + + # --- GarageConfig --- + + @testset "GarageConfig construction" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds) + @test config.endpoint == "https://s3-api.example.com" + @test config.region == "us-east-1" + @test config.credentials === creds + @test config isa AWS.AbstractAWSConfig + end + + @testset "GarageConfig AWS extensions" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) + @test AWS.region(config) == "garage" + @test AWS.credentials(config) === creds + end + + # --- AWS.generate_service_url --- + + @testset "generate_service_url with resource starting with /" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/mybucket/key") + @test url == "https://s3-api.example.com/mybucket/key" + end + + @testset "generate_service_url with resource NOT starting with /" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) + url = AWS.generate_service_url(config, "s3", "mybucket/key") + @test url == "https://s3-api.example.com/mybucket/key" + end + + @testset "generate_service_url strips trailing slashes from endpoint" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "https://s3-api.example.com/bucket/key" + end + + @testset "generate_service_url with multiple trailing slashes" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "https://s3-api.example.com/bucket/key" + end + + @testset "generate_service_url preserves http endpoint" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "http://localhost:3900/bucket/key" + end + + @testset "generate_service_url with http endpoint and trailing slash" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "http://localhost:3900/bucket/key" + end + + # --- GarageStorage --- + + @testset "GarageStorage default region" begin + storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket") + @test storage.bucket == "mybucket" + @test storage.config.region == "garage" + @test storage.config.endpoint == "https://s3-api.example.com" + end + + @testset "GarageStorage custom region" begin + storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region") + @test storage.config.region == "custom-region" + end + + @testset "GarageStorage endpoint without https" begin + storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket") + @test storage.config.endpoint == "http://localhost:3900" + end + + @testset "GarageStorage credentials are SimpleCredentials" begin + storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket") + @test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials + @test storage.config.credentials.access_key_id == "mykey" + @test storage.config.credentials.secret_key == "mysecret" + @test storage.config.credentials.token == "" + end + + @testset "GarageStorage returns correct struct type" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + @test storage isa GarageStorage + end + + # --- put_file callable struct (slash detection) --- + + @testset "put_file callable struct returns nothing for keys with slashes" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + uploader = put_file(storage) + result = @test_logs (:warn, r"Upload aborted") uploader("a/b.json", "data") + @test result === nothing + end + + @testset "put_file callable struct type is correct" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + uploader = put_file(storage) + @test uploader isa put_file + end + + @testset "put_file callable struct construction stores storage" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + uploader = put_file(storage) + @test uploader.storage === storage + end + + # --- put_file function (slash detection) --- + + @testset "put_file function returns nothing for keys with slashes" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + result = @test_logs (:warn, r"Upload aborted") put_file(storage, "a/b.json", "data") + @test result === nothing + end + + @testset "put_file function returns nothing for keys with multiple slashes" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + result = @test_logs (:warn, r"Upload aborted") put_file(storage, "a/b/c/d.json", "data") + @test result === nothing + end + + @testset "put_file function returns nothing for key with single slash in middle" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + result = @test_logs (:warn, r"Upload aborted") put_file(storage, "folder/file.json", "data") + @test result === nothing + end + + @testset "put_file function returns nothing for key with trailing slash pattern" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + result = @test_logs (:warn, r"Upload aborted") put_file(storage, "a/b", "data") + @test result === nothing + end + + # --- Integration tests with real Garage server --- + + @testset "integration: full CRUD lifecycle" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-$(randstring(8)).json" + test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" + + # Test put_file(storage, key, data) - successful upload + result = put_file(storage, test_key, test_data) + @test result !== nothing + @test haskey(result, :api) + @test haskey(result, :web) + @test occursin("testbucket", result.api) + @test occursin("s3-api", result.api) + @test occursin("s3-web", result.web) + + # Test get_file - retrieve the uploaded data + downloaded = get_file(storage, test_key) + @test downloaded !== nothing + @test downloaded isa Vector{UInt8} + @test String(downloaded) == test_data + + # Test list_files - verify key appears in listing + keys = list_files(storage) + @test test_key in keys + + # Test put_file callable struct + test_key2 = "testrunner-callable-$(randstring(8)).json" + callable_result = put_file(storage)(test_key2, "callable-data") + @test callable_result !== nothing + @test callable_result.api != "" + @test callable_result.web != "" + + # Verify callable upload + downloaded2 = get_file(storage, test_key2) + @test String(downloaded2) == "callable-data" + + # Test delete_file + delete_file(storage, test_key) + deleted_data = get_file(storage, test_key) + @test deleted_data === nothing + + # Cleanup callable test object + delete_file(storage, test_key2) + deleted_data2 = get_file(storage, test_key2) + @test deleted_data2 === nothing + end + + @testset "integration: put_file rejects slashes even with real server" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + result = @test_logs (:warn, r"Upload aborted") put_file(storage, "folder/test.json", "data") + @test result === nothing + end + + @testset "integration: get_file returns nothing for missing key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + result = get_file(storage, "nonexistent-key-$(randstring(8)).json") + @test result === nothing + end + + @testset "integration: binary data upload and download" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-binary-$(randstring(8)).bin" + binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] + + result = put_file(storage, test_key, binary_data) + @test result !== nothing + + downloaded = get_file(storage, test_key) + @test downloaded == binary_data + + delete_file(storage, test_key) + @test get_file(storage, test_key) === nothing + end + + @testset "integration: list_files returns vector of strings" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-list-$(randstring(8)).txt" + put_file(storage, test_key, "list-test-data") + + keys = list_files(storage) + @test keys isa Vector{String} + @test test_key in keys + + delete_file(storage, test_key) + end + + @testset "integration: url format validation" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-url-$(randstring(8)).json" + result = put_file(storage, test_key, "url-test") + + @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") + @test occursin("/$test_key", result.api) + @test startswith(result.web, "https://s3-web.yiem.cc/testbucket/") + @test occursin("/$test_key", result.web) + + delete_file(storage, test_key) + end -@testset "detect_keyword tests" begin - @test detect_keyword(["test"], "this is a test") == Dict("test" => 1) - - @test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1) - - @test detect_keyword(["cat"], "category") == Dict("cat" => 1) - - @test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0) - - @test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0) - - @test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1) - - @test detect_keyword(["word"], "") == Dict("word" => 0) - - @test detect_keyword(String[], "some text") == Dict{String, Integer}() - - @test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1) - - multiline_text = """ - first line - second line - first word - """ - @test detect_keyword(["first"], multiline_text) == Dict("first" => 2) - - @test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1) - - @test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1) end - - - - - -