Merge pull request 'V0.6.2' (#14) from v0.6.2 into main
Reviewed-on: #14
This commit was merged in pull request #14.
This commit is contained in:
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
name = "GeneralUtils"
|
name = "GeneralUtils"
|
||||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||||
version = "0.6.1"
|
version = "0.6.2"
|
||||||
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
||||||
|
|
||||||
[deps]
|
[deps]
|
||||||
|
|||||||
@@ -1,271 +0,0 @@
|
|||||||
using SHA
|
|
||||||
using Dates
|
|
||||||
using URIs
|
|
||||||
using OrderedCollections
|
|
||||||
using AWS
|
|
||||||
@service S3
|
|
||||||
using AWSS3
|
|
||||||
|
|
||||||
|
|
||||||
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
|
||||||
# this file use local AWS config
|
|
||||||
|
|
||||||
""" Example
|
|
||||||
storage = GeneralUtils.GarageStorage(
|
|
||||||
"https://s3-api.yiem.cc",
|
|
||||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
|
||||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
|
||||||
"sommpanion-s3"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
|
||||||
GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
|
|
||||||
keys = GeneralUtils.list_files(storage)
|
|
||||||
|
|
||||||
println("Read back: ", data)
|
|
||||||
println("Bucket keys: ", keys)
|
|
||||||
|
|
||||||
# test with curl
|
|
||||||
curl -v \
|
|
||||||
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
|
||||||
http://192.168.88.106:3902/users-1002.json
|
|
||||||
|
|
||||||
# test with API
|
|
||||||
data = String(GeneralUtils.get_file(storage, "users-1005.json"))
|
|
||||||
|
|
||||||
# test with a browser
|
|
||||||
https://s3-web.mydomain.com/my_bucket_name/users-1005.json
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# ===================================================================
|
|
||||||
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
|
||||||
# ===================================================================
|
|
||||||
|
|
||||||
struct SimpleCredentials
|
|
||||||
access_key_id::String
|
|
||||||
secret_key::String
|
|
||||||
token::String
|
|
||||||
end
|
|
||||||
|
|
||||||
struct GarageConfig <: AWS.AbstractAWSConfig
|
|
||||||
endpoint::String
|
|
||||||
region::String
|
|
||||||
credentials::SimpleCredentials
|
|
||||||
end
|
|
||||||
|
|
||||||
# Required extensions for AWS.jl pipeline
|
|
||||||
AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
|
|
||||||
AWS.credentials(c::SimpleCredentials) = c
|
|
||||||
AWS.check_credentials(c::SimpleCredentials) = c
|
|
||||||
AWS.region(aws::GarageConfig) = aws.region
|
|
||||||
AWS.credentials(aws::GarageConfig) = aws.credentials
|
|
||||||
|
|
||||||
""" Generate the base URL for a Garage S3 service request.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `aws::GarageConfig`
|
|
||||||
The Garage configuration containing the endpoint.
|
|
||||||
- `service::String`
|
|
||||||
The AWS service name (e.g., `"s3"`).
|
|
||||||
- `region::String`
|
|
||||||
The AWS region string.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `String`: The stripped endpoint URL (trailing slashes removed).
|
|
||||||
"""
|
|
||||||
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
|
||||||
return strip(aws.endpoint, '/')
|
|
||||||
end
|
|
||||||
|
|
||||||
""" Generate the full service URL including the resource path for a Garage S3 request.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `aws::GarageConfig`
|
|
||||||
The Garage configuration containing the endpoint.
|
|
||||||
- `service::String`
|
|
||||||
The AWS service name (e.g., `"s3"`).
|
|
||||||
- `resource::String`
|
|
||||||
The S3 resource path (e.g., `"/bucket/key"`).
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `String`: The full URL combining the endpoint and resource path.
|
|
||||||
"""
|
|
||||||
function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
|
|
||||||
endpoint = strip(aws.endpoint, '/')
|
|
||||||
resource_path = startswith(resource, '/') ? resource : "/" * resource
|
|
||||||
return string(endpoint, resource_path)
|
|
||||||
end
|
|
||||||
|
|
||||||
# ===================================================================
|
|
||||||
# 2. Thread-Safe Storage Client Wrapper
|
|
||||||
# ===================================================================
|
|
||||||
|
|
||||||
""" Create a thread-safe Garage S3 storage client.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `endpoint::String`
|
|
||||||
The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
|
|
||||||
- `key::String`
|
|
||||||
The Garage access key ID (created via garage-ui).
|
|
||||||
- `secret::String`
|
|
||||||
The secret key corresponding to the access key.
|
|
||||||
- `bucket::String`
|
|
||||||
The name of the S3 bucket to use.
|
|
||||||
|
|
||||||
# Keyword Arguments
|
|
||||||
- `region::String = "garage"`
|
|
||||||
The region identifier for the Garage instance.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `GarageStorage`: A thread-safe storage client instance.
|
|
||||||
|
|
||||||
# Examples
|
|
||||||
```jldoctest
|
|
||||||
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
|
||||||
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
struct GarageStorage
|
|
||||||
config::GarageConfig
|
|
||||||
bucket::String
|
|
||||||
end
|
|
||||||
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
|
||||||
creds = SimpleCredentials(key, secret, "")
|
|
||||||
config = GarageConfig(endpoint, region, creds)
|
|
||||||
return GarageStorage(config, bucket)
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl
|
|
||||||
function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}})
|
|
||||||
return SHA.hmac_sha256(key, msg)
|
|
||||||
end
|
|
||||||
|
|
||||||
function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}})
|
|
||||||
return SHA.hmac_sha256(Vector{UInt8}(key), msg)
|
|
||||||
end
|
|
||||||
|
|
||||||
# 2. AWSS3 overload targeting your custom Garage endpoint
|
|
||||||
function AWSS3._s3_sign_url_v4(
|
|
||||||
aws::GarageConfig,
|
|
||||||
bucket::String,
|
|
||||||
path::String,
|
|
||||||
seconds::Int=3600;
|
|
||||||
verb::String="GET",
|
|
||||||
content_type::String="application/octet-stream",
|
|
||||||
protocol::String="https",
|
|
||||||
)
|
|
||||||
path_escaped = URIs.escapepath("/$bucket/$path")
|
|
||||||
|
|
||||||
now_datetime = now(Dates.UTC)
|
|
||||||
datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
|
|
||||||
date_stamp = Dates.format(now_datetime, "YYYYmmdd")
|
|
||||||
|
|
||||||
service = "s3"
|
|
||||||
scheme = "AWS4"
|
|
||||||
algorithm = "HMAC-SHA256"
|
|
||||||
terminator = "aws4_request"
|
|
||||||
|
|
||||||
# Robust host & port extraction (prevents trailing colon bugs like 'domain.com:')
|
|
||||||
parsed_uri = URIs.URI(aws.endpoint)
|
|
||||||
has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0
|
|
||||||
host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host
|
|
||||||
effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme
|
|
||||||
|
|
||||||
scope = "$date_stamp/$(aws.region)/$service/$terminator"
|
|
||||||
|
|
||||||
headers = OrderedDict{String,String}("Host" => host)
|
|
||||||
sort!(headers; by=name -> lowercase(name))
|
|
||||||
canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
|
|
||||||
|
|
||||||
query = OrderedDict{String,String}(
|
|
||||||
"X-Amz-Expires" => string(seconds),
|
|
||||||
"X-Amz-Algorithm" => "$scheme-$algorithm",
|
|
||||||
"X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope",
|
|
||||||
"X-Amz-Date" => datetime_stamp,
|
|
||||||
"X-Amz-SignedHeaders" => canonical_header_names,
|
|
||||||
)
|
|
||||||
|
|
||||||
if !isempty(aws.credentials.token)
|
|
||||||
query["X-Amz-Security-Token"] = aws.credentials.token
|
|
||||||
end
|
|
||||||
|
|
||||||
sort!(query; by=name -> lowercase(name))
|
|
||||||
|
|
||||||
canonical_headers = join(
|
|
||||||
map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers))
|
|
||||||
)
|
|
||||||
|
|
||||||
canonical_request = string(
|
|
||||||
"$verb\n",
|
|
||||||
"$path_escaped\n",
|
|
||||||
"$(URIs.escapeuri(query))\n",
|
|
||||||
"$canonical_headers\n",
|
|
||||||
"$canonical_header_names\n",
|
|
||||||
"UNSIGNED-PAYLOAD",
|
|
||||||
)
|
|
||||||
|
|
||||||
hashed_canonical_request = bytes2hex(sha256(canonical_request))
|
|
||||||
|
|
||||||
string_to_sign = string(
|
|
||||||
"$scheme-$algorithm\n",
|
|
||||||
"$datetime_stamp\n",
|
|
||||||
"$scope\n",
|
|
||||||
hashed_canonical_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
# AWS Signature V4 Key Derivation Chain
|
|
||||||
key_secret = string(scheme, aws.credentials.secret_key)
|
|
||||||
key_date = _hmac_sha256(key_secret, date_stamp)
|
|
||||||
key_region = _hmac_sha256(key_date, aws.region)
|
|
||||||
key_service = _hmac_sha256(key_region, service)
|
|
||||||
key_signing = _hmac_sha256(key_service, terminator)
|
|
||||||
signature = _hmac_sha256(key_signing, string_to_sign)
|
|
||||||
|
|
||||||
query["X-Amz-Signature"] = bytes2hex(signature)
|
|
||||||
|
|
||||||
return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query))
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
|
|
||||||
|
|
||||||
Generates direct public HTTP GET URLs via Garage's s3-web interface.
|
|
||||||
Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint.
|
|
||||||
"""
|
|
||||||
function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
|
|
||||||
clean_endpoint = strip(web_endpoint, '/')
|
|
||||||
clean_key = lstrip(key, '/')
|
|
||||||
|
|
||||||
# s3-web URL format: https://<web-domain>/<bucket>/<key>
|
|
||||||
download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)"
|
|
||||||
|
|
||||||
return (
|
|
||||||
download_url = download_url,
|
|
||||||
# DELETE requires s3-api (s3-web does not support HTTP DELETE)
|
|
||||||
delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE")
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# 1. Initialize Garage configuration pointing to S3 API domain
|
|
||||||
storage = GarageStorage(
|
|
||||||
"https://s3-api.yiem.cc",
|
|
||||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
|
||||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
|
||||||
"sommpanion-s3"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 2. Upload microservice file payload
|
|
||||||
put_file(storage, "job-9042-data.bin", payload_bytes)
|
|
||||||
|
|
||||||
# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes)
|
|
||||||
urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900)
|
|
||||||
|
|
||||||
# Pass `urls.download_url` and `urls.delete_url` to the downstream service
|
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# dummy main.jl so i can do GeneralUtils precompile
|
||||||
|
using GeneralUtils
|
||||||
+112
-53
@@ -1,6 +1,7 @@
|
|||||||
module garageS3
|
module garageS3
|
||||||
|
|
||||||
export
|
export
|
||||||
|
LifecycleExpiration, build_lifecycle_xml,
|
||||||
GarageStorage, put_file, get_file, list_files, delete_file,
|
GarageStorage, put_file, get_file, list_files, delete_file,
|
||||||
set_lifecycle_expiration, get_lifecycle, delete_lifecycle
|
set_lifecycle_expiration, get_lifecycle, delete_lifecycle
|
||||||
|
|
||||||
@@ -13,11 +14,11 @@ using AWSS3
|
|||||||
# this file use local AWS config
|
# this file use local AWS config
|
||||||
|
|
||||||
""" Example
|
""" Example
|
||||||
storage = GeneralUtils.GarageStorage(
|
storage1 = GeneralUtils.GarageStorage(
|
||||||
"https://s3-api.yiem.cc",
|
"https://s3-api.yiem.cc",
|
||||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
||||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
||||||
"sommpanion-s3"
|
"testbucket"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
||||||
@@ -30,7 +31,7 @@ println("Bucket keys: ", key)
|
|||||||
|
|
||||||
# test with curl
|
# test with curl
|
||||||
curl -v \
|
curl -v \
|
||||||
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
-H 'Host: testbucket.s3-web.yiem.cc' \
|
||||||
http://192.168.88.106:3902/users-1002.json
|
http://192.168.88.106:3902/users-1002.json
|
||||||
|
|
||||||
# test with API
|
# test with API
|
||||||
@@ -64,23 +65,6 @@ AWS.check_credentials(c::SimpleCredentials) = c
|
|||||||
AWS.region(aws::GarageConfig) = aws.region
|
AWS.region(aws::GarageConfig) = aws.region
|
||||||
AWS.credentials(aws::GarageConfig) = aws.credentials
|
AWS.credentials(aws::GarageConfig) = aws.credentials
|
||||||
|
|
||||||
""" Generate the base URL for a Garage S3 service request.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `aws::GarageConfig`
|
|
||||||
The Garage configuration containing the endpoint.
|
|
||||||
- `service::String`
|
|
||||||
The AWS service name (e.g., `"s3"`).
|
|
||||||
- `region::String`
|
|
||||||
The AWS region string.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `String`: The stripped endpoint URL (trailing slashes removed).
|
|
||||||
"""
|
|
||||||
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
|
||||||
return strip(aws.endpoint, '/')
|
|
||||||
end
|
|
||||||
|
|
||||||
""" Generate the full service URL including the resource path for a Garage S3 request.
|
""" Generate the full service URL including the resource path for a Garage S3 request.
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
@@ -143,17 +127,12 @@ end
|
|||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
- `key::String`
|
- `key::String`
|
||||||
The object key (name) in the bucket. Must not contain slashes (`/`).
|
The object key (name) in the bucket.
|
||||||
- `data::Union{String, Vector{UInt8}}`
|
- `data::Union{String, Vector{UInt8}}`
|
||||||
The data to upload.
|
The data to upload.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs if the upload succeeds.
|
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
|
||||||
- `nothing` if the key contains slashes (upload aborted with warning).
|
|
||||||
|
|
||||||
# Notes
|
|
||||||
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
|
||||||
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
|
||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
```jldoctest
|
```jldoctest
|
||||||
@@ -171,19 +150,12 @@ struct put_file
|
|||||||
storage::GarageStorage
|
storage::GarageStorage
|
||||||
end
|
end
|
||||||
function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}}
|
function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}}
|
||||||
)::Union{NamedTuple{(:api, :web), Tuple{String, String}}, Nothing}
|
)::NamedTuple{(:api, :web), Tuple{String, String}}
|
||||||
# Check if the key contains a slash (virtual folder character)
|
|
||||||
if occursin('/', key)
|
|
||||||
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
|
||||||
return nothing
|
|
||||||
end
|
|
||||||
|
|
||||||
# Proceed if the key is flat
|
|
||||||
AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
|
AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
|
||||||
println("Successfully uploaded: ", key)
|
println("Successfully uploaded: ", key)
|
||||||
|
|
||||||
# object url
|
# object url
|
||||||
api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
|
api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
|
||||||
|
|
||||||
# file download URL using browser
|
# file download URL using browser
|
||||||
web = replace(api, "s3-api" => "s3-web")
|
web = replace(api, "s3-api" => "s3-web")
|
||||||
@@ -197,17 +169,12 @@ end
|
|||||||
- `storage::GarageStorage`
|
- `storage::GarageStorage`
|
||||||
The storage client instance.
|
The storage client instance.
|
||||||
- `key::String`
|
- `key::String`
|
||||||
The object key (name) in the bucket. Must not contain slashes (`/`).
|
The object key (name) in the bucket.
|
||||||
- `data::Union{String, Vector{UInt8}}`
|
- `data::Union{String, Vector{UInt8}}`
|
||||||
The data to upload.
|
The data to upload.
|
||||||
|
|
||||||
# Return
|
# Return
|
||||||
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs if the upload succeeds.
|
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
|
||||||
- `nothing` if the key contains slashes (upload aborted with warning).
|
|
||||||
|
|
||||||
# Notes
|
|
||||||
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
|
||||||
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
|
||||||
|
|
||||||
# Examples
|
# Examples
|
||||||
```julia
|
```julia
|
||||||
@@ -220,14 +187,7 @@ julia> result.web
|
|||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
|
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
|
||||||
)::Union{NamedTuple{(:api, :web), Tuple{String, String}}, Nothing}
|
)::NamedTuple{(:api, :web), Tuple{String, String}}
|
||||||
# Check if the key contains a slash (virtual folder character)
|
|
||||||
if occursin('/', key)
|
|
||||||
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
|
||||||
return nothing
|
|
||||||
end
|
|
||||||
|
|
||||||
# Proceed if the key is flat
|
|
||||||
AWSS3.s3_put(storage.config, storage.bucket, key, data)
|
AWSS3.s3_put(storage.config, storage.bucket, key, data)
|
||||||
println("Successfully uploaded: ", key)
|
println("Successfully uploaded: ", key)
|
||||||
|
|
||||||
@@ -284,8 +244,8 @@ julia> keys = list_files(storage)
|
|||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
function list_files(storage::GarageStorage)
|
function list_files(storage::GarageStorage)
|
||||||
# Approach 2: s3_list_objects returns a Vector of Dicts with object details
|
# Use delimiter="" to get all individual keys (no grouping by "directory")
|
||||||
objects = AWSS3.s3_list_objects(storage.config, storage.bucket)
|
objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="")
|
||||||
return [obj["Key"] for obj in objects]
|
return [obj["Key"] for obj in objects]
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -307,11 +267,110 @@ function delete_file(storage::GarageStorage, key::String)
|
|||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
|
# ===================================================================
|
||||||
|
# 3. Bucket Lifecycle Configuration
|
||||||
|
# ===================================================================
|
||||||
|
|
||||||
|
""" Lifecycle expiration configuration for a bucket.
|
||||||
|
|
||||||
|
# Fields
|
||||||
|
- `prefix::String`: Object key prefix this rule applies to.
|
||||||
|
- `days::Int`: Number of days after object creation to expire/delete.
|
||||||
|
- `enabled::Bool`: Whether the rule is active (default: `true`).
|
||||||
|
- `id::String`: Unique identifier for the rule (default: auto-generated).
|
||||||
|
"""
|
||||||
|
struct LifecycleExpiration
|
||||||
|
prefix::String
|
||||||
|
days::Int
|
||||||
|
enabled::Bool
|
||||||
|
id::String
|
||||||
|
end
|
||||||
|
function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="")
|
||||||
|
rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id
|
||||||
|
return LifecycleExpiration(prefix, days, enabled, rule_id)
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Build the XML body for a PutBucketLifecycleConfiguration request.
|
||||||
|
|
||||||
|
Supports a single rule with expiration by prefix.
|
||||||
|
"""
|
||||||
|
function build_lifecycle_xml(rule::LifecycleExpiration)::String
|
||||||
|
status = rule.enabled ? "Enabled" : "Disabled"
|
||||||
|
return """<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>$(rule.id)</ID>
|
||||||
|
<Filter>
|
||||||
|
<Prefix>$(rule.prefix)</Prefix>
|
||||||
|
</Filter>
|
||||||
|
<Status>$status</Status>
|
||||||
|
<Expiration>
|
||||||
|
<Days>$(rule.days)</Days>
|
||||||
|
</Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>"""
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Set bucket lifecycle expiration rule.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`: The storage client instance.
|
||||||
|
- `rule::LifecycleExpiration`: The lifecycle expiration rule.
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- `Dict{String, Any}`: Parsed response from the server.
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
rule = LifecycleExpiration("uploads/", 30)
|
||||||
|
set_lifecycle_expiration(storage, rule)
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration)
|
||||||
|
xml_body = build_lifecycle_xml(rule)
|
||||||
|
args = Dict{String,Any}("body" => xml_body)
|
||||||
|
return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config)
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Get the bucket lifecycle configuration.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`: The storage client instance.
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured.
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
config = get_lifecycle(storage)
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
|
||||||
|
try
|
||||||
|
return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config)
|
||||||
|
catch e
|
||||||
|
if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration"
|
||||||
|
return nothing
|
||||||
|
end
|
||||||
|
rethrow(e)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
""" Delete the bucket lifecycle configuration.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `storage::GarageStorage`: The storage client instance.
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- `Dict{String, Any}`: Parsed response from the server.
|
||||||
|
|
||||||
|
# Example
|
||||||
|
```julia
|
||||||
|
delete_lifecycle(storage)
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
function delete_lifecycle(storage::GarageStorage)
|
||||||
|
return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config)
|
||||||
|
end
|
||||||
|
|
||||||
end # module GarageS3
|
end # module GarageS3
|
||||||
|
|||||||
+452
-28
@@ -1,39 +1,463 @@
|
|||||||
using Test
|
using Test
|
||||||
using GeneralUtils: detect_keyword
|
using GeneralUtils
|
||||||
|
using GeneralUtils.garageS3
|
||||||
|
using AWS
|
||||||
|
using Random: randstring
|
||||||
|
using UUIDs: uuid4
|
||||||
|
|
||||||
@testset "detect_keyword tests" begin
|
@testset "garageS3.jl" begin
|
||||||
@test detect_keyword(["test"], "this is a test") == Dict("test" => 1)
|
|
||||||
|
|
||||||
@test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1)
|
# --- SimpleCredentials ---
|
||||||
|
|
||||||
@test detect_keyword(["cat"], "category") == Dict("cat" => 1)
|
@testset "SimpleCredentials construction" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken")
|
||||||
@test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0)
|
@test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE"
|
||||||
|
@test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||||
@test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0)
|
@test creds.token == "sessiontoken"
|
||||||
|
|
||||||
@test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1)
|
|
||||||
|
|
||||||
@test detect_keyword(["word"], "") == Dict("word" => 0)
|
|
||||||
|
|
||||||
@test detect_keyword(String[], "some text") == Dict{String, Integer}()
|
|
||||||
|
|
||||||
@test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1)
|
|
||||||
|
|
||||||
multiline_text = """
|
|
||||||
first line
|
|
||||||
second line
|
|
||||||
first word
|
|
||||||
"""
|
|
||||||
@test detect_keyword(["first"], multiline_text) == Dict("first" => 2)
|
|
||||||
|
|
||||||
@test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1)
|
|
||||||
|
|
||||||
@test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@testset "SimpleCredentials AWS extensions" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1")
|
||||||
|
@test AWS.credentials(creds) === creds
|
||||||
|
@test AWS.check_credentials(creds) === creds
|
||||||
|
@test AWS.refresh!(creds; force=false) === creds
|
||||||
|
@test AWS.refresh!(creds) === creds
|
||||||
|
end
|
||||||
|
|
||||||
|
# --- GarageConfig ---
|
||||||
|
|
||||||
|
@testset "GarageConfig construction" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds)
|
||||||
|
@test config.endpoint == "https://s3-api.example.com"
|
||||||
|
@test config.region == "us-east-1"
|
||||||
|
@test config.credentials === creds
|
||||||
|
@test config isa AWS.AbstractAWSConfig
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "GarageConfig AWS extensions" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||||
|
@test AWS.region(config) == "garage"
|
||||||
|
@test AWS.credentials(config) === creds
|
||||||
|
end
|
||||||
|
|
||||||
|
# --- AWS.generate_service_url ---
|
||||||
|
|
||||||
|
@testset "generate_service_url with resource starting with /" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||||
|
url = AWS.generate_service_url(config, "s3", "/mybucket/key")
|
||||||
|
@test url == "https://s3-api.example.com/mybucket/key"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "generate_service_url with resource NOT starting with /" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||||
|
url = AWS.generate_service_url(config, "s3", "mybucket/key")
|
||||||
|
@test url == "https://s3-api.example.com/mybucket/key"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "generate_service_url strips trailing slashes from endpoint" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds)
|
||||||
|
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||||
|
@test url == "https://s3-api.example.com/bucket/key"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "generate_service_url with multiple trailing slashes" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds)
|
||||||
|
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||||
|
@test url == "https://s3-api.example.com/bucket/key"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "generate_service_url preserves http endpoint" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds)
|
||||||
|
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||||
|
@test url == "http://localhost:3900/bucket/key"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "generate_service_url with http endpoint and trailing slash" begin
|
||||||
|
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||||
|
config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds)
|
||||||
|
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||||
|
@test url == "http://localhost:3900/bucket/key"
|
||||||
|
end
|
||||||
|
|
||||||
|
# --- GarageStorage ---
|
||||||
|
|
||||||
|
@testset "GarageStorage default region" begin
|
||||||
|
storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket")
|
||||||
|
@test storage.bucket == "mybucket"
|
||||||
|
@test storage.config.region == "garage"
|
||||||
|
@test storage.config.endpoint == "https://s3-api.example.com"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "GarageStorage custom region" begin
|
||||||
|
storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region")
|
||||||
|
@test storage.config.region == "custom-region"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "GarageStorage endpoint without https" begin
|
||||||
|
storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket")
|
||||||
|
@test storage.config.endpoint == "http://localhost:3900"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "GarageStorage credentials are SimpleCredentials" begin
|
||||||
|
storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket")
|
||||||
|
@test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials
|
||||||
|
@test storage.config.credentials.access_key_id == "mykey"
|
||||||
|
@test storage.config.credentials.secret_key == "mysecret"
|
||||||
|
@test storage.config.credentials.token == ""
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "GarageStorage returns correct struct type" begin
|
||||||
|
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||||
|
@test storage isa GarageStorage
|
||||||
|
end
|
||||||
|
|
||||||
|
# --- put_file callable struct ---
|
||||||
|
|
||||||
|
@testset "put_file callable struct type is correct" begin
|
||||||
|
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||||
|
uploader = put_file(storage)
|
||||||
|
@test uploader isa put_file
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "put_file callable struct construction stores storage" begin
|
||||||
|
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||||
|
uploader = put_file(storage)
|
||||||
|
@test uploader.storage === storage
|
||||||
|
end
|
||||||
|
|
||||||
|
# --- Integration tests with real Garage server ---
|
||||||
|
|
||||||
|
@testset "integration: full CRUD lifecycle" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "testrunner-$(randstring(8)).json"
|
||||||
|
test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}"
|
||||||
|
|
||||||
|
# Test put_file(storage, key, data) - successful upload
|
||||||
|
result = put_file(storage, test_key, test_data)
|
||||||
|
@test result !== nothing
|
||||||
|
@test haskey(result, :api)
|
||||||
|
@test haskey(result, :web)
|
||||||
|
@test occursin("testbucket", result.api)
|
||||||
|
@test occursin("s3-api", result.api)
|
||||||
|
@test occursin("s3-web", result.web)
|
||||||
|
|
||||||
|
# Test get_file - retrieve the uploaded data
|
||||||
|
downloaded = get_file(storage, test_key)
|
||||||
|
@test downloaded !== nothing
|
||||||
|
@test downloaded isa Vector{UInt8}
|
||||||
|
@test String(downloaded) == test_data
|
||||||
|
|
||||||
|
# Test list_files - verify key appears in listing
|
||||||
|
keys = list_files(storage)
|
||||||
|
@test test_key in keys
|
||||||
|
|
||||||
|
# Test put_file callable struct
|
||||||
|
test_key2 = "testrunner-callable-$(randstring(8)).json"
|
||||||
|
callable_result = put_file(storage)(test_key2, "callable-data")
|
||||||
|
@test callable_result !== nothing
|
||||||
|
@test callable_result.api != ""
|
||||||
|
@test callable_result.web != ""
|
||||||
|
|
||||||
|
# Verify callable upload
|
||||||
|
downloaded2 = get_file(storage, test_key2)
|
||||||
|
@test String(downloaded2) == "callable-data"
|
||||||
|
|
||||||
|
# Test delete_file
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
deleted_data = get_file(storage, test_key)
|
||||||
|
@test deleted_data === nothing
|
||||||
|
|
||||||
|
# Cleanup callable test object
|
||||||
|
delete_file(storage, test_key2)
|
||||||
|
deleted_data2 = get_file(storage, test_key2)
|
||||||
|
@test deleted_data2 === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: nested key CRUD - single level" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "single/slash/test-$(randstring(8)).json"
|
||||||
|
test_data = "{\"level\": 1, \"nested\": true}"
|
||||||
|
|
||||||
|
# Upload nested key
|
||||||
|
result = put_file(storage, test_key, test_data)
|
||||||
|
@test result isa NamedTuple
|
||||||
|
@test haskey(result, :api)
|
||||||
|
@test haskey(result, :web)
|
||||||
|
@test occursin("single/slash/", result.api)
|
||||||
|
@test occursin("single/slash/", result.web)
|
||||||
|
|
||||||
|
# Download and verify
|
||||||
|
downloaded = get_file(storage, test_key)
|
||||||
|
@test downloaded !== nothing
|
||||||
|
@test String(downloaded) == test_data
|
||||||
|
|
||||||
|
# Verify in listing
|
||||||
|
keys = list_files(storage)
|
||||||
|
@test test_key in keys
|
||||||
|
|
||||||
|
# Delete and verify gone
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
@test get_file(storage, test_key) === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: nested key CRUD - multi level" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "deep/nested/folder/subfolder/test-$(randstring(8)).json"
|
||||||
|
test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}"
|
||||||
|
|
||||||
|
# Upload deeply nested key
|
||||||
|
result = put_file(storage, test_key, test_data)
|
||||||
|
@test result isa NamedTuple
|
||||||
|
@test occursin("deep/nested/folder/subfolder/", result.api)
|
||||||
|
@test occursin("s3-api", result.api)
|
||||||
|
@test occursin("s3-web", result.web)
|
||||||
|
|
||||||
|
# Download and verify
|
||||||
|
downloaded = get_file(storage, test_key)
|
||||||
|
@test downloaded !== nothing
|
||||||
|
@test String(downloaded) == test_data
|
||||||
|
|
||||||
|
# Verify in listing
|
||||||
|
keys = list_files(storage)
|
||||||
|
@test test_key in keys
|
||||||
|
|
||||||
|
# Delete and verify gone
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
@test get_file(storage, test_key) === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: nested key with callable struct" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "callable/nested/test-$(randstring(8)).json"
|
||||||
|
test_data = "{\"callable\": true, \"nested\": true}"
|
||||||
|
|
||||||
|
uploader = put_file(storage)
|
||||||
|
result = uploader(test_key, test_data)
|
||||||
|
@test result isa NamedTuple
|
||||||
|
@test occursin("callable/nested/", result.api)
|
||||||
|
|
||||||
|
downloaded = get_file(storage, test_key)
|
||||||
|
@test String(downloaded) == test_data
|
||||||
|
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
@test get_file(storage, test_key) === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: get_file returns nothing for missing key" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
result = get_file(storage, "nonexistent-key-$(randstring(8)).json")
|
||||||
|
@test result === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: binary data upload and download" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "testrunner-binary-$(randstring(8)).bin"
|
||||||
|
binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f]
|
||||||
|
|
||||||
|
result = put_file(storage, test_key, binary_data)
|
||||||
|
@test result !== nothing
|
||||||
|
|
||||||
|
downloaded = get_file(storage, test_key)
|
||||||
|
@test downloaded == binary_data
|
||||||
|
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
@test get_file(storage, test_key) === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: list_files returns vector of strings" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "testrunner-list-$(randstring(8)).txt"
|
||||||
|
put_file(storage, test_key, "list-test-data")
|
||||||
|
|
||||||
|
keys = list_files(storage)
|
||||||
|
@test keys isa Vector{String}
|
||||||
|
@test test_key in keys
|
||||||
|
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: url format validation" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
test_key = "testrunner-url-$(randstring(8)).json"
|
||||||
|
result = put_file(storage, test_key, "url-test")
|
||||||
|
|
||||||
|
@test startswith(result.api, "https://s3-api.yiem.cc/testbucket/")
|
||||||
|
@test occursin("/$test_key", result.api)
|
||||||
|
@test startswith(result.web, "https://s3-web.yiem.cc/testbucket/")
|
||||||
|
@test occursin("/$test_key", result.web)
|
||||||
|
|
||||||
|
delete_file(storage, test_key)
|
||||||
|
end
|
||||||
|
|
||||||
|
# --- Lifecycle configuration tests ---
|
||||||
|
|
||||||
|
@testset "LifecycleExpiration construction with default ID" begin
|
||||||
|
rule = LifecycleExpiration("uploads/", 30)
|
||||||
|
@test rule.prefix == "uploads/"
|
||||||
|
@test rule.days == 30
|
||||||
|
@test rule.enabled == true
|
||||||
|
@test rule.id == "expire-uploads_-30d"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "LifecycleExpiration construction with custom ID" begin
|
||||||
|
rule = LifecycleExpiration("data/", 7; id="my-rule")
|
||||||
|
@test rule.prefix == "data/"
|
||||||
|
@test rule.days == 7
|
||||||
|
@test rule.enabled == true
|
||||||
|
@test rule.id == "my-rule"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "LifecycleExpiration with disabled flag" begin
|
||||||
|
rule = LifecycleExpiration("temp/", 1; enabled=false)
|
||||||
|
@test rule.enabled == false
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "LifecycleExpiration with nested prefix" begin
|
||||||
|
rule = LifecycleExpiration("deep/nested/", 90)
|
||||||
|
@test rule.prefix == "deep/nested/"
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "build_lifecycle_xml generates valid XML" begin
|
||||||
|
rule = LifecycleExpiration("test/", 14; id="test-rule")
|
||||||
|
xml = build_lifecycle_xml(rule)
|
||||||
|
@test occursin("<?xml version=\"1.0\"", xml)
|
||||||
|
@test occursin("<LifecycleConfiguration>", xml)
|
||||||
|
@test occursin("<ID>test-rule</ID>", xml)
|
||||||
|
@test occursin("<Prefix>test/</Prefix>", xml)
|
||||||
|
@test occursin("<Status>Enabled</Status>", xml)
|
||||||
|
@test occursin("<Days>14</Days>", xml)
|
||||||
|
@test occursin("</LifecycleConfiguration>", xml)
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "build_lifecycle_xml generates disabled rule XML" begin
|
||||||
|
rule = LifecycleExpiration("temp/", 1; enabled=false)
|
||||||
|
xml = build_lifecycle_xml(rule)
|
||||||
|
@test occursin("<Status>Disabled</Status>", xml)
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: lifecycle CRUD - set and get" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Should return nothing when no lifecycle is set
|
||||||
|
lc = get_lifecycle(storage)
|
||||||
|
@test lc === nothing
|
||||||
|
|
||||||
|
# Set lifecycle
|
||||||
|
rule = LifecycleExpiration("lifecycle-test/", 45)
|
||||||
|
set_lifecycle_expiration(storage, rule)
|
||||||
|
|
||||||
|
# Should return configuration after setting
|
||||||
|
lc = get_lifecycle(storage)
|
||||||
|
@test lc !== nothing
|
||||||
|
@test haskey(lc, "Rule")
|
||||||
|
|
||||||
|
# Delete lifecycle
|
||||||
|
delete_lifecycle(storage)
|
||||||
|
|
||||||
|
# Should return nothing after deletion
|
||||||
|
lc = get_lifecycle(storage)
|
||||||
|
@test lc === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: lifecycle with nested prefix" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule")
|
||||||
|
set_lifecycle_expiration(storage, rule)
|
||||||
|
|
||||||
|
lc = get_lifecycle(storage)
|
||||||
|
@test lc !== nothing
|
||||||
|
@test haskey(lc, "Rule")
|
||||||
|
|
||||||
|
delete_lifecycle(storage)
|
||||||
|
@test get_lifecycle(storage) === nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
@testset "integration: lifecycle enabled/disabled toggle" begin
|
||||||
|
storage = GarageStorage(
|
||||||
|
"https://s3-api.yiem.cc",
|
||||||
|
"GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"testbucket"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Set enabled rule
|
||||||
|
rule1 = LifecycleExpiration("toggle/", 10; enabled=true)
|
||||||
|
set_lifecycle_expiration(storage, rule1)
|
||||||
|
lc = get_lifecycle(storage)
|
||||||
|
@test lc !== nothing
|
||||||
|
|
||||||
|
# Override with disabled rule
|
||||||
|
rule2 = LifecycleExpiration("toggle/", 10; enabled=false)
|
||||||
|
set_lifecycle_expiration(storage, rule2)
|
||||||
|
lc = get_lifecycle(storage)
|
||||||
|
@test lc !== nothing
|
||||||
|
|
||||||
|
delete_lifecycle(storage)
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|||||||
Reference in New Issue
Block a user