377 lines
10 KiB
Julia
377 lines
10 KiB
Julia
module garageS3
|
|
|
|
export
|
|
LifecycleExpiration, build_lifecycle_xml,
|
|
GarageStorage, put_file, get_file, list_files, delete_file,
|
|
set_lifecycle_expiration, get_lifecycle, delete_lifecycle
|
|
|
|
using AWS
|
|
@service S3
|
|
using AWSS3
|
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
|
|
|
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
|
# this file use local AWS config
|
|
|
|
""" Example
|
|
storage1 = GeneralUtils.GarageStorage(
|
|
"https://s3-api.yiem.cc",
|
|
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
|
"testbucket"
|
|
)
|
|
|
|
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
|
# use UUID as filename because GarageS3 requires unique filename for each uploaded object
|
|
urls = GeneralUtils.put_file(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
|
|
key = GeneralUtils.list_files(storage)
|
|
|
|
println("Read back: ", data)
|
|
println("Bucket keys: ", key)
|
|
|
|
# test with curl
|
|
curl -v \
|
|
-H 'Host: testbucket.s3-web.yiem.cc' \
|
|
http://192.168.88.106:3902/users-1002.json
|
|
|
|
# test with API
|
|
data = String(GeneralUtils.get_file(storage, key[1]))
|
|
|
|
# test with a browser (urls.web)
|
|
https://s3-web.mydomain.com/my_bucket_name/users-1005.json
|
|
|
|
"""
|
|
|
|
# ===================================================================
|
|
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
|
# ===================================================================
|
|
|
|
struct SimpleCredentials
|
|
access_key_id::String
|
|
secret_key::String
|
|
token::String
|
|
end
|
|
|
|
struct GarageConfig <: AWS.AbstractAWSConfig
|
|
endpoint::String
|
|
region::String
|
|
credentials::SimpleCredentials
|
|
end
|
|
|
|
# Required extensions for AWS.jl pipeline
|
|
AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
|
|
AWS.credentials(c::SimpleCredentials) = c
|
|
AWS.check_credentials(c::SimpleCredentials) = c
|
|
AWS.region(aws::GarageConfig) = aws.region
|
|
AWS.credentials(aws::GarageConfig) = aws.credentials
|
|
|
|
""" Generate the full service URL including the resource path for a Garage S3 request.
|
|
|
|
# Arguments
|
|
- `aws::GarageConfig`
|
|
The Garage configuration containing the endpoint.
|
|
- `service::String`
|
|
The AWS service name (e.g., `"s3"`).
|
|
- `resource::String`
|
|
The S3 resource path (e.g., `"/bucket/key"`).
|
|
|
|
# Return
|
|
- `String`: The full URL combining the endpoint and resource path.
|
|
"""
|
|
function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
|
|
endpoint = strip(aws.endpoint, '/')
|
|
resource_path = startswith(resource, '/') ? resource : "/" * resource
|
|
return string(endpoint, resource_path)
|
|
end
|
|
|
|
# ===================================================================
|
|
# 2. Thread-Safe Storage Client Wrapper
|
|
# ===================================================================
|
|
|
|
""" Create a thread-safe Garage S3 storage client.
|
|
|
|
# Arguments
|
|
- `endpoint::String`
|
|
The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
|
|
- `key::String`
|
|
The Garage access key ID (created via garage-ui).
|
|
- `secret::String`
|
|
The secret key corresponding to the access key.
|
|
- `bucket::String`
|
|
The name of the S3 bucket to use.
|
|
|
|
# Keyword Arguments
|
|
- `region::String = "garage"`
|
|
The region identifier for the Garage instance.
|
|
|
|
# Return
|
|
- `GarageStorage`: A thread-safe storage client instance.
|
|
|
|
# Examples
|
|
```jldoctest
|
|
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
|
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
|
```
|
|
"""
|
|
struct GarageStorage
|
|
config::GarageConfig
|
|
bucket::String
|
|
end
|
|
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
|
creds = SimpleCredentials(key, secret, "")
|
|
config = GarageConfig(endpoint, region, creds)
|
|
return GarageStorage(config, bucket)
|
|
end
|
|
|
|
""" Upload an object to the Garage S3 bucket (callable struct).
|
|
|
|
# Arguments
|
|
- `key::String`
|
|
The object key (name) in the bucket.
|
|
- `data::Union{String, Vector{UInt8}}`
|
|
The data to upload.
|
|
|
|
# Return
|
|
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
|
|
|
|
# Examples
|
|
```jldoctest
|
|
julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
|
julia> put_file1 = put_file(storage1)
|
|
julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}")
|
|
Successfully uploaded: test.json
|
|
julia> downloadUrl.api
|
|
"https://s3-api.yiem.cc/sommpanion-s3/test.json"
|
|
julia> downloadUrl.web
|
|
"https://s3-web.yiem.cc/sommpanion-s3/test.json"
|
|
```
|
|
"""
|
|
struct put_file
|
|
storage::GarageStorage
|
|
end
|
|
function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}}
|
|
)::NamedTuple{(:api, :web), Tuple{String, String}}
|
|
AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
|
|
println("Successfully uploaded: ", key)
|
|
|
|
# object url
|
|
api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
|
|
|
|
# file download URL using browser
|
|
web = replace(api, "s3-api" => "s3-web")
|
|
|
|
return (api=api, web=web)
|
|
end
|
|
|
|
""" Upload an object to the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
- `key::String`
|
|
The object key (name) in the bucket.
|
|
- `data::Union{String, Vector{UInt8}}`
|
|
The data to upload.
|
|
|
|
# Return
|
|
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}")
|
|
Successfully uploaded: test.json
|
|
julia> result.api
|
|
"https://s3-api.yiem.cc/sommpanion-s3/test.json"
|
|
julia> result.web
|
|
"https://s3-web.yiem.cc/sommpanion-s3/test.json"
|
|
```
|
|
"""
|
|
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
|
|
)::NamedTuple{(:api, :web), Tuple{String, String}}
|
|
AWSS3.s3_put(storage.config, storage.bucket, key, data)
|
|
println("Successfully uploaded: ", key)
|
|
|
|
# object url
|
|
api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
|
|
|
|
# file download URL using browser
|
|
web = replace(api, "s3-api" => "s3-web")
|
|
|
|
return (api=api, web=web)
|
|
end
|
|
|
|
""" Download an object from the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
- `key::String`
|
|
The object key (name) in the bucket.
|
|
|
|
# Return
|
|
- `Vector{UInt8}`: The raw bytes of the downloaded object.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> data = String(get_file(storage, "test.json"))
|
|
"\"{\\\"status\\\": \\\"active\\\"}\""
|
|
```
|
|
"""
|
|
function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
|
|
try
|
|
return AWSS3.s3_get(storage.config, storage.bucket, key)
|
|
catch
|
|
return nothing
|
|
end
|
|
end
|
|
|
|
""" List all object keys in the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
|
|
# Return
|
|
- `Vector{String}`: A list of object keys (file names) in the bucket.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> keys = list_files(storage)
|
|
3-element Vector{String}:
|
|
"test1.json"
|
|
"test2.json"
|
|
"users-1002.json"
|
|
```
|
|
"""
|
|
function list_files(storage::GarageStorage)
|
|
# Use delimiter="" to get all individual keys (no grouping by "directory")
|
|
objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="")
|
|
return [obj["Key"] for obj in objects]
|
|
end
|
|
|
|
""" Delete an object from the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
- `key::String`
|
|
The object key (name) to delete.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> delete_file(storage, "test.json")
|
|
```
|
|
"""
|
|
function delete_file(storage::GarageStorage, key::String)
|
|
AWSS3.s3_delete(storage.config, storage.bucket, key)
|
|
end
|
|
|
|
|
|
# ===================================================================
|
|
# 3. Bucket Lifecycle Configuration
|
|
# ===================================================================
|
|
|
|
""" Lifecycle expiration configuration for a bucket.
|
|
|
|
# Fields
|
|
- `prefix::String`: Object key prefix this rule applies to.
|
|
- `days::Int`: Number of days after object creation to expire/delete.
|
|
- `enabled::Bool`: Whether the rule is active (default: `true`).
|
|
- `id::String`: Unique identifier for the rule (default: auto-generated).
|
|
"""
|
|
struct LifecycleExpiration
|
|
prefix::String
|
|
days::Int
|
|
enabled::Bool
|
|
id::String
|
|
end
|
|
function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="")
|
|
rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id
|
|
return LifecycleExpiration(prefix, days, enabled, rule_id)
|
|
end
|
|
|
|
""" Build the XML body for a PutBucketLifecycleConfiguration request.
|
|
|
|
Supports a single rule with expiration by prefix.
|
|
"""
|
|
function build_lifecycle_xml(rule::LifecycleExpiration)::String
|
|
status = rule.enabled ? "Enabled" : "Disabled"
|
|
return """<?xml version="1.0" encoding="UTF-8"?>
|
|
<LifecycleConfiguration>
|
|
<Rule>
|
|
<ID>$(rule.id)</ID>
|
|
<Filter>
|
|
<Prefix>$(rule.prefix)</Prefix>
|
|
</Filter>
|
|
<Status>$status</Status>
|
|
<Expiration>
|
|
<Days>$(rule.days)</Days>
|
|
</Expiration>
|
|
</Rule>
|
|
</LifecycleConfiguration>"""
|
|
end
|
|
|
|
""" Set bucket lifecycle expiration rule.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`: The storage client instance.
|
|
- `rule::LifecycleExpiration`: The lifecycle expiration rule.
|
|
|
|
# Returns
|
|
- `Dict{String, Any}`: Parsed response from the server.
|
|
|
|
# Example
|
|
```julia
|
|
rule = LifecycleExpiration("uploads/", 30)
|
|
set_lifecycle_expiration(storage, rule)
|
|
```
|
|
"""
|
|
function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration)
|
|
xml_body = build_lifecycle_xml(rule)
|
|
args = Dict{String,Any}("body" => xml_body)
|
|
return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config)
|
|
end
|
|
|
|
""" Get the bucket lifecycle configuration.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`: The storage client instance.
|
|
|
|
# Returns
|
|
- `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured.
|
|
|
|
# Example
|
|
```julia
|
|
config = get_lifecycle(storage)
|
|
```
|
|
"""
|
|
function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
|
|
try
|
|
return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config)
|
|
catch e
|
|
if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration"
|
|
return nothing
|
|
end
|
|
rethrow(e)
|
|
end
|
|
end
|
|
|
|
""" Delete the bucket lifecycle configuration.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`: The storage client instance.
|
|
|
|
# Returns
|
|
- `Dict{String, Any}`: Parsed response from the server.
|
|
|
|
# Example
|
|
```julia
|
|
delete_lifecycle(storage)
|
|
```
|
|
"""
|
|
function delete_lifecycle(storage::GarageStorage)
|
|
return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config)
|
|
end
|
|
|
|
end # module GarageS3
|