diff --git a/Project.toml b/Project.toml index 829d4a0..d798a3c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -version = "0.6.1" +version = "0.6.2" authors = ["tonaerospace "] [deps] diff --git a/etc.jl b/etc.jl deleted file mode 100644 index 70bf716..0000000 --- a/etc.jl +++ /dev/null @@ -1,271 +0,0 @@ -using SHA -using Dates -using URIs -using OrderedCollections -using AWS -@service S3 -using AWSS3 - - -# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key). -# this file use local AWS config - -""" Example -storage = GeneralUtils.GarageStorage( - "https://s3-api.yiem.cc", - "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) - "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "sommpanion-s3" -) - -# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) -GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}") -keys = GeneralUtils.list_files(storage) - -println("Read back: ", data) -println("Bucket keys: ", keys) - -# test with curl -curl -v \ - -H 'Host: sommpanion-s3.s3-web.yiem.cc' \ - http://192.168.88.106:3902/users-1002.json - -# test with API -data = String(GeneralUtils.get_file(storage, "users-1005.json")) - -# test with a browser -https://s3-web.mydomain.com/my_bucket_name/users-1005.json - -""" - -# =================================================================== -# 1. Custom GarageConfig Definition (Thread-Safe, No Global State) -# =================================================================== - -struct SimpleCredentials - access_key_id::String - secret_key::String - token::String -end - -struct GarageConfig <: AWS.AbstractAWSConfig - endpoint::String - region::String - credentials::SimpleCredentials -end - -# Required extensions for AWS.jl pipeline -AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c -AWS.credentials(c::SimpleCredentials) = c -AWS.check_credentials(c::SimpleCredentials) = c -AWS.region(aws::GarageConfig) = aws.region -AWS.credentials(aws::GarageConfig) = aws.credentials - -""" Generate the base URL for a Garage S3 service request. - -# Arguments -- `aws::GarageConfig` - The Garage configuration containing the endpoint. -- `service::String` - The AWS service name (e.g., `"s3"`). -- `region::String` - The AWS region string. - -# Return -- `String`: The stripped endpoint URL (trailing slashes removed). -""" -function AWS.generate_service_url(aws::GarageConfig, service::String, region::String) - return strip(aws.endpoint, '/') -end - -""" Generate the full service URL including the resource path for a Garage S3 request. - -# Arguments -- `aws::GarageConfig` - The Garage configuration containing the endpoint. -- `service::String` - The AWS service name (e.g., `"s3"`). -- `resource::String` - The S3 resource path (e.g., `"/bucket/key"`). - -# Return -- `String`: The full URL combining the endpoint and resource path. -""" -function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String) - endpoint = strip(aws.endpoint, '/') - resource_path = startswith(resource, '/') ? resource : "/" * resource - return string(endpoint, resource_path) -end - -# =================================================================== -# 2. Thread-Safe Storage Client Wrapper -# =================================================================== - -""" Create a thread-safe Garage S3 storage client. - -# Arguments -- `endpoint::String` - The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`). -- `key::String` - The Garage access key ID (created via garage-ui). -- `secret::String` - The secret key corresponding to the access key. -- `bucket::String` - The name of the S3 bucket to use. - -# Keyword Arguments -- `region::String = "garage"` - The region identifier for the Garage instance. - -# Return -- `GarageStorage`: A thread-safe storage client instance. - -# Examples -```jldoctest -julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") -GarageStorage(GarageConfig(...), "sommpanion-s3") -``` -""" -struct GarageStorage - config::GarageConfig - bucket::String -end -function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage") - creds = SimpleCredentials(key, secret, "") - config = GarageConfig(endpoint, region, creds) - return GarageStorage(config, bucket) -end - - -# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl -function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}}) - return SHA.hmac_sha256(key, msg) -end - -function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}}) - return SHA.hmac_sha256(Vector{UInt8}(key), msg) -end - -# 2. AWSS3 overload targeting your custom Garage endpoint -function AWSS3._s3_sign_url_v4( - aws::GarageConfig, - bucket::String, - path::String, - seconds::Int=3600; - verb::String="GET", - content_type::String="application/octet-stream", - protocol::String="https", -) - path_escaped = URIs.escapepath("/$bucket/$path") - - now_datetime = now(Dates.UTC) - datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") - date_stamp = Dates.format(now_datetime, "YYYYmmdd") - - service = "s3" - scheme = "AWS4" - algorithm = "HMAC-SHA256" - terminator = "aws4_request" - - # Robust host & port extraction (prevents trailing colon bugs like 'domain.com:') - parsed_uri = URIs.URI(aws.endpoint) - has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0 - host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host - effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme - - scope = "$date_stamp/$(aws.region)/$service/$terminator" - - headers = OrderedDict{String,String}("Host" => host) - sort!(headers; by=name -> lowercase(name)) - canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") - - query = OrderedDict{String,String}( - "X-Amz-Expires" => string(seconds), - "X-Amz-Algorithm" => "$scheme-$algorithm", - "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope", - "X-Amz-Date" => datetime_stamp, - "X-Amz-SignedHeaders" => canonical_header_names, - ) - - if !isempty(aws.credentials.token) - query["X-Amz-Security-Token"] = aws.credentials.token - end - - sort!(query; by=name -> lowercase(name)) - - canonical_headers = join( - map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers)) - ) - - canonical_request = string( - "$verb\n", - "$path_escaped\n", - "$(URIs.escapeuri(query))\n", - "$canonical_headers\n", - "$canonical_header_names\n", - "UNSIGNED-PAYLOAD", - ) - - hashed_canonical_request = bytes2hex(sha256(canonical_request)) - - string_to_sign = string( - "$scheme-$algorithm\n", - "$datetime_stamp\n", - "$scope\n", - hashed_canonical_request, - ) - - # AWS Signature V4 Key Derivation Chain - key_secret = string(scheme, aws.credentials.secret_key) - key_date = _hmac_sha256(key_secret, date_stamp) - key_region = _hmac_sha256(key_date, aws.region) - key_service = _hmac_sha256(key_region, service) - key_signing = _hmac_sha256(key_service, terminator) - signature = _hmac_sha256(key_signing, string_to_sign) - - query["X-Amz-Signature"] = bytes2hex(signature) - - return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query)) -end - - - -""" - generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") - -Generates direct public HTTP GET URLs via Garage's s3-web interface. -Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint. -""" -function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") - clean_endpoint = strip(web_endpoint, '/') - clean_key = lstrip(key, '/') - - # s3-web URL format: https://// - download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)" - - return ( - download_url = download_url, - # DELETE requires s3-api (s3-web does not support HTTP DELETE) - delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE") - ) -end - - - - - -# 1. Initialize Garage configuration pointing to S3 API domain -storage = GarageStorage( - "https://s3-api.yiem.cc", - "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) - "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "sommpanion-s3" -) - -# 2. Upload microservice file payload -put_file(storage, "job-9042-data.bin", payload_bytes) - -# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes) -urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900) - -# Pass `urls.download_url` and `urls.delete_url` to the downstream service \ No newline at end of file diff --git a/main.jl b/main.jl new file mode 100644 index 0000000..95cc800 --- /dev/null +++ b/main.jl @@ -0,0 +1,2 @@ +# dummy main.jl so i can do GeneralUtils precompile +using GeneralUtils \ No newline at end of file diff --git a/src/garageS3.jl b/src/garageS3.jl index 7086955..ca264b6 100644 --- a/src/garageS3.jl +++ b/src/garageS3.jl @@ -1,6 +1,7 @@ module garageS3 export + LifecycleExpiration, build_lifecycle_xml, GarageStorage, put_file, get_file, list_files, delete_file, set_lifecycle_expiration, get_lifecycle, delete_lifecycle @@ -13,11 +14,11 @@ using AWSS3 # this file use local AWS config """ Example -storage = GeneralUtils.GarageStorage( +storage1 = GeneralUtils.GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "sommpanion-s3" + "testbucket" ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) @@ -30,7 +31,7 @@ println("Bucket keys: ", key) # test with curl curl -v \ - -H 'Host: sommpanion-s3.s3-web.yiem.cc' \ + -H 'Host: testbucket.s3-web.yiem.cc' \ http://192.168.88.106:3902/users-1002.json # test with API @@ -64,23 +65,6 @@ AWS.check_credentials(c::SimpleCredentials) = c AWS.region(aws::GarageConfig) = aws.region AWS.credentials(aws::GarageConfig) = aws.credentials -""" Generate the base URL for a Garage S3 service request. - -# Arguments -- `aws::GarageConfig` - The Garage configuration containing the endpoint. -- `service::String` - The AWS service name (e.g., `"s3"`). -- `region::String` - The AWS region string. - -# Return -- `String`: The stripped endpoint URL (trailing slashes removed). -""" -function AWS.generate_service_url(aws::GarageConfig, service::String, region::String) - return strip(aws.endpoint, '/') -end - """ Generate the full service URL including the resource path for a Garage S3 request. # Arguments @@ -143,17 +127,12 @@ end # Arguments - `key::String` - The object key (name) in the bucket. Must not contain slashes (`/`). + The object key (name) in the bucket. - `data::Union{String, Vector{UInt8}}` The data to upload. # Return -- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs if the upload succeeds. -- `nothing` if the key contains slashes (upload aborted with warning). - -# Notes -- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming - convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`). +- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs. # Examples ```jldoctest @@ -171,19 +150,12 @@ struct put_file storage::GarageStorage end function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}} - )::Union{NamedTuple{(:api, :web), Tuple{String, String}}, Nothing} - # Check if the key contains a slash (virtual folder character) - if occursin('/', key) - @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')." - return nothing - end - - # Proceed if the key is flat + )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) println("Successfully uploaded: ", key) # object url - api = "$(storage.config.endpoint)/$(storage.bucket)/$key" + api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key" # file download URL using browser web = replace(api, "s3-api" => "s3-web") @@ -197,17 +169,12 @@ end - `storage::GarageStorage` The storage client instance. - `key::String` - The object key (name) in the bucket. Must not contain slashes (`/`). + The object key (name) in the bucket. - `data::Union{String, Vector{UInt8}}` The data to upload. # Return -- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs if the upload succeeds. -- `nothing` if the key contains slashes (upload aborted with warning). - -# Notes -- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming - convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`). +- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs. # Examples ```julia @@ -220,14 +187,7 @@ julia> result.web ``` """ function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} - )::Union{NamedTuple{(:api, :web), Tuple{String, String}}, Nothing} - # Check if the key contains a slash (virtual folder character) - if occursin('/', key) - @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')." - return nothing - end - - # Proceed if the key is flat + )::NamedTuple{(:api, :web), Tuple{String, String}} AWSS3.s3_put(storage.config, storage.bucket, key, data) println("Successfully uploaded: ", key) @@ -284,8 +244,8 @@ julia> keys = list_files(storage) ``` """ function list_files(storage::GarageStorage) - # Approach 2: s3_list_objects returns a Vector of Dicts with object details - objects = AWSS3.s3_list_objects(storage.config, storage.bucket) + # Use delimiter="" to get all individual keys (no grouping by "directory") + objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="") return [obj["Key"] for obj in objects] end @@ -307,11 +267,110 @@ function delete_file(storage::GarageStorage, key::String) end +# =================================================================== +# 3. Bucket Lifecycle Configuration +# =================================================================== +""" Lifecycle expiration configuration for a bucket. +# Fields +- `prefix::String`: Object key prefix this rule applies to. +- `days::Int`: Number of days after object creation to expire/delete. +- `enabled::Bool`: Whether the rule is active (default: `true`). +- `id::String`: Unique identifier for the rule (default: auto-generated). +""" +struct LifecycleExpiration + prefix::String + days::Int + enabled::Bool + id::String +end +function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="") + rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id + return LifecycleExpiration(prefix, days, enabled, rule_id) +end +""" Build the XML body for a PutBucketLifecycleConfiguration request. +Supports a single rule with expiration by prefix. +""" +function build_lifecycle_xml(rule::LifecycleExpiration)::String + status = rule.enabled ? "Enabled" : "Disabled" + return """ + + + $(rule.id) + + $(rule.prefix) + + $status + + $(rule.days) + + +""" +end +""" Set bucket lifecycle expiration rule. +# Arguments +- `storage::GarageStorage`: The storage client instance. +- `rule::LifecycleExpiration`: The lifecycle expiration rule. + +# Returns +- `Dict{String, Any}`: Parsed response from the server. + +# Example +```julia +rule = LifecycleExpiration("uploads/", 30) +set_lifecycle_expiration(storage, rule) +``` +""" +function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration) + xml_body = build_lifecycle_xml(rule) + args = Dict{String,Any}("body" => xml_body) + return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config) +end + +""" Get the bucket lifecycle configuration. + +# Arguments +- `storage::GarageStorage`: The storage client instance. + +# Returns +- `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured. + +# Example +```julia +config = get_lifecycle(storage) +``` +""" +function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} + try + return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config) + catch e + if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration" + return nothing + end + rethrow(e) + end +end + +""" Delete the bucket lifecycle configuration. + +# Arguments +- `storage::GarageStorage`: The storage client instance. + +# Returns +- `Dict{String, Any}`: Parsed response from the server. + +# Example +```julia +delete_lifecycle(storage) +``` +""" +function delete_lifecycle(storage::GarageStorage) + return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config) +end end # module GarageS3 diff --git a/test/runtests.jl b/test/runtests.jl index f415525..8ef2cf7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,39 +1,463 @@ using Test -using GeneralUtils: detect_keyword +using GeneralUtils +using GeneralUtils.garageS3 +using AWS +using Random: randstring +using UUIDs: uuid4 + +@testset "garageS3.jl" begin + + # --- SimpleCredentials --- + + @testset "SimpleCredentials construction" begin + creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken") + @test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE" + @test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + @test creds.token == "sessiontoken" + end + + @testset "SimpleCredentials AWS extensions" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1") + @test AWS.credentials(creds) === creds + @test AWS.check_credentials(creds) === creds + @test AWS.refresh!(creds; force=false) === creds + @test AWS.refresh!(creds) === creds + end + + # --- GarageConfig --- + + @testset "GarageConfig construction" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds) + @test config.endpoint == "https://s3-api.example.com" + @test config.region == "us-east-1" + @test config.credentials === creds + @test config isa AWS.AbstractAWSConfig + end + + @testset "GarageConfig AWS extensions" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) + @test AWS.region(config) == "garage" + @test AWS.credentials(config) === creds + end + + # --- AWS.generate_service_url --- + + @testset "generate_service_url with resource starting with /" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/mybucket/key") + @test url == "https://s3-api.example.com/mybucket/key" + end + + @testset "generate_service_url with resource NOT starting with /" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds) + url = AWS.generate_service_url(config, "s3", "mybucket/key") + @test url == "https://s3-api.example.com/mybucket/key" + end + + @testset "generate_service_url strips trailing slashes from endpoint" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "https://s3-api.example.com/bucket/key" + end + + @testset "generate_service_url with multiple trailing slashes" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "https://s3-api.example.com/bucket/key" + end + + @testset "generate_service_url preserves http endpoint" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "http://localhost:3900/bucket/key" + end + + @testset "generate_service_url with http endpoint and trailing slash" begin + creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "") + config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds) + url = AWS.generate_service_url(config, "s3", "/bucket/key") + @test url == "http://localhost:3900/bucket/key" + end + + # --- GarageStorage --- + + @testset "GarageStorage default region" begin + storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket") + @test storage.bucket == "mybucket" + @test storage.config.region == "garage" + @test storage.config.endpoint == "https://s3-api.example.com" + end + + @testset "GarageStorage custom region" begin + storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region") + @test storage.config.region == "custom-region" + end + + @testset "GarageStorage endpoint without https" begin + storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket") + @test storage.config.endpoint == "http://localhost:3900" + end + + @testset "GarageStorage credentials are SimpleCredentials" begin + storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket") + @test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials + @test storage.config.credentials.access_key_id == "mykey" + @test storage.config.credentials.secret_key == "mysecret" + @test storage.config.credentials.token == "" + end + + @testset "GarageStorage returns correct struct type" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + @test storage isa GarageStorage + end + + # --- put_file callable struct --- + + @testset "put_file callable struct type is correct" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + uploader = put_file(storage) + @test uploader isa put_file + end + + @testset "put_file callable struct construction stores storage" begin + storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket") + uploader = put_file(storage) + @test uploader.storage === storage + end + + # --- Integration tests with real Garage server --- + + @testset "integration: full CRUD lifecycle" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-$(randstring(8)).json" + test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" + + # Test put_file(storage, key, data) - successful upload + result = put_file(storage, test_key, test_data) + @test result !== nothing + @test haskey(result, :api) + @test haskey(result, :web) + @test occursin("testbucket", result.api) + @test occursin("s3-api", result.api) + @test occursin("s3-web", result.web) + + # Test get_file - retrieve the uploaded data + downloaded = get_file(storage, test_key) + @test downloaded !== nothing + @test downloaded isa Vector{UInt8} + @test String(downloaded) == test_data + + # Test list_files - verify key appears in listing + keys = list_files(storage) + @test test_key in keys + + # Test put_file callable struct + test_key2 = "testrunner-callable-$(randstring(8)).json" + callable_result = put_file(storage)(test_key2, "callable-data") + @test callable_result !== nothing + @test callable_result.api != "" + @test callable_result.web != "" + + # Verify callable upload + downloaded2 = get_file(storage, test_key2) + @test String(downloaded2) == "callable-data" + + # Test delete_file + delete_file(storage, test_key) + deleted_data = get_file(storage, test_key) + @test deleted_data === nothing + + # Cleanup callable test object + delete_file(storage, test_key2) + deleted_data2 = get_file(storage, test_key2) + @test deleted_data2 === nothing + end + + @testset "integration: nested key CRUD - single level" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "single/slash/test-$(randstring(8)).json" + test_data = "{\"level\": 1, \"nested\": true}" + + # Upload nested key + result = put_file(storage, test_key, test_data) + @test result isa NamedTuple + @test haskey(result, :api) + @test haskey(result, :web) + @test occursin("single/slash/", result.api) + @test occursin("single/slash/", result.web) + + # Download and verify + downloaded = get_file(storage, test_key) + @test downloaded !== nothing + @test String(downloaded) == test_data + + # Verify in listing + keys = list_files(storage) + @test test_key in keys + + # Delete and verify gone + delete_file(storage, test_key) + @test get_file(storage, test_key) === nothing + end + + @testset "integration: nested key CRUD - multi level" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "deep/nested/folder/subfolder/test-$(randstring(8)).json" + test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}" + + # Upload deeply nested key + result = put_file(storage, test_key, test_data) + @test result isa NamedTuple + @test occursin("deep/nested/folder/subfolder/", result.api) + @test occursin("s3-api", result.api) + @test occursin("s3-web", result.web) + + # Download and verify + downloaded = get_file(storage, test_key) + @test downloaded !== nothing + @test String(downloaded) == test_data + + # Verify in listing + keys = list_files(storage) + @test test_key in keys + + # Delete and verify gone + delete_file(storage, test_key) + @test get_file(storage, test_key) === nothing + end + + @testset "integration: nested key with callable struct" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "callable/nested/test-$(randstring(8)).json" + test_data = "{\"callable\": true, \"nested\": true}" + + uploader = put_file(storage) + result = uploader(test_key, test_data) + @test result isa NamedTuple + @test occursin("callable/nested/", result.api) + + downloaded = get_file(storage, test_key) + @test String(downloaded) == test_data + + delete_file(storage, test_key) + @test get_file(storage, test_key) === nothing + end + + @testset "integration: get_file returns nothing for missing key" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + result = get_file(storage, "nonexistent-key-$(randstring(8)).json") + @test result === nothing + end + + @testset "integration: binary data upload and download" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-binary-$(randstring(8)).bin" + binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] + + result = put_file(storage, test_key, binary_data) + @test result !== nothing + + downloaded = get_file(storage, test_key) + @test downloaded == binary_data + + delete_file(storage, test_key) + @test get_file(storage, test_key) === nothing + end + + @testset "integration: list_files returns vector of strings" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-list-$(randstring(8)).txt" + put_file(storage, test_key, "list-test-data") + + keys = list_files(storage) + @test keys isa Vector{String} + @test test_key in keys + + delete_file(storage, test_key) + end + + @testset "integration: url format validation" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + test_key = "testrunner-url-$(randstring(8)).json" + result = put_file(storage, test_key, "url-test") + + @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") + @test occursin("/$test_key", result.api) + @test startswith(result.web, "https://s3-web.yiem.cc/testbucket/") + @test occursin("/$test_key", result.web) + + delete_file(storage, test_key) + end + + # --- Lifecycle configuration tests --- + + @testset "LifecycleExpiration construction with default ID" begin + rule = LifecycleExpiration("uploads/", 30) + @test rule.prefix == "uploads/" + @test rule.days == 30 + @test rule.enabled == true + @test rule.id == "expire-uploads_-30d" + end + + @testset "LifecycleExpiration construction with custom ID" begin + rule = LifecycleExpiration("data/", 7; id="my-rule") + @test rule.prefix == "data/" + @test rule.days == 7 + @test rule.enabled == true + @test rule.id == "my-rule" + end + + @testset "LifecycleExpiration with disabled flag" begin + rule = LifecycleExpiration("temp/", 1; enabled=false) + @test rule.enabled == false + end + + @testset "LifecycleExpiration with nested prefix" begin + rule = LifecycleExpiration("deep/nested/", 90) + @test rule.prefix == "deep/nested/" + end + + @testset "build_lifecycle_xml generates valid XML" begin + rule = LifecycleExpiration("test/", 14; id="test-rule") + xml = build_lifecycle_xml(rule) + @test occursin("", xml) + @test occursin("test-rule", xml) + @test occursin("test/", xml) + @test occursin("Enabled", xml) + @test occursin("14", xml) + @test occursin("", xml) + end + + @testset "build_lifecycle_xml generates disabled rule XML" begin + rule = LifecycleExpiration("temp/", 1; enabled=false) + xml = build_lifecycle_xml(rule) + @test occursin("Disabled", xml) + end + + @testset "integration: lifecycle CRUD - set and get" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + # Should return nothing when no lifecycle is set + lc = get_lifecycle(storage) + @test lc === nothing + + # Set lifecycle + rule = LifecycleExpiration("lifecycle-test/", 45) + set_lifecycle_expiration(storage, rule) + + # Should return configuration after setting + lc = get_lifecycle(storage) + @test lc !== nothing + @test haskey(lc, "Rule") + + # Delete lifecycle + delete_lifecycle(storage) + + # Should return nothing after deletion + lc = get_lifecycle(storage) + @test lc === nothing + end + + @testset "integration: lifecycle with nested prefix" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule") + set_lifecycle_expiration(storage, rule) + + lc = get_lifecycle(storage) + @test lc !== nothing + @test haskey(lc, "Rule") + + delete_lifecycle(storage) + @test get_lifecycle(storage) === nothing + end + + @testset "integration: lifecycle enabled/disabled toggle" begin + storage = GarageStorage( + "https://s3-api.yiem.cc", + "GKb080154a2e5b19100b1b2c6e", + "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", + "testbucket" + ) + + # Set enabled rule + rule1 = LifecycleExpiration("toggle/", 10; enabled=true) + set_lifecycle_expiration(storage, rule1) + lc = get_lifecycle(storage) + @test lc !== nothing + + # Override with disabled rule + rule2 = LifecycleExpiration("toggle/", 10; enabled=false) + set_lifecycle_expiration(storage, rule2) + lc = get_lifecycle(storage) + @test lc !== nothing + + delete_lifecycle(storage) + end -@testset "detect_keyword tests" begin - @test detect_keyword(["test"], "this is a test") == Dict("test" => 1) - - @test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1) - - @test detect_keyword(["cat"], "category") == Dict("cat" => 1) - - @test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0) - - @test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0) - - @test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1) - - @test detect_keyword(["word"], "") == Dict("word" => 0) - - @test detect_keyword(String[], "some text") == Dict{String, Integer}() - - @test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1) - - multiline_text = """ - first line - second line - first word - """ - @test detect_keyword(["first"], multiline_text) == Dict("first" => 2) - - @test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1) - - @test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1) end - - - - - -