307 lines
8.3 KiB
Julia
307 lines
8.3 KiB
Julia
module garageS3
|
|
|
|
export
|
|
GarageStorage,
|
|
put_file,
|
|
get_file,
|
|
list_files,
|
|
delete_file
|
|
|
|
using AWS, AWSS3
|
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
|
|
|
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
|
# this file use local AWS config
|
|
|
|
""" Example
|
|
storage = GarageStorage(
|
|
"https://s3-api.yiem.cc",
|
|
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
|
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
|
"sommpanion-s3"
|
|
)
|
|
|
|
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
|
put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
|
|
keys = list_files(storage)
|
|
data = String(get_file(storage, "users-1004.json"))
|
|
|
|
println("Read back: ", data)
|
|
println("Bucket keys: ", keys)
|
|
|
|
# test with curl
|
|
curl -v \
|
|
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
|
http://192.168.88.106:3902/users-1002.json
|
|
|
|
# test with API
|
|
data = String(get_file(storage, "users-1005.json"))
|
|
|
|
# test with a browser
|
|
https://s3-web.mydomain.com/my_bucket_name/users-1005.json
|
|
|
|
"""
|
|
|
|
# ===================================================================
|
|
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
|
# ===================================================================
|
|
|
|
struct SimpleCredentials
|
|
access_key_id::String
|
|
secret_key::String
|
|
token::String
|
|
end
|
|
|
|
struct GarageConfig <: AWS.AbstractAWSConfig
|
|
endpoint::String
|
|
region::String
|
|
credentials::SimpleCredentials
|
|
end
|
|
|
|
# Required extensions for AWS.jl pipeline
|
|
AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
|
|
AWS.credentials(c::SimpleCredentials) = c
|
|
AWS.check_credentials(c::SimpleCredentials) = c
|
|
AWS.region(aws::GarageConfig) = aws.region
|
|
AWS.credentials(aws::GarageConfig) = aws.credentials
|
|
|
|
""" Generate the base URL for a Garage S3 service request.
|
|
|
|
# Arguments
|
|
- `aws::GarageConfig`
|
|
The Garage configuration containing the endpoint.
|
|
- `service::String`
|
|
The AWS service name (e.g., `"s3"`).
|
|
- `region::String`
|
|
The AWS region string.
|
|
|
|
# Return
|
|
- `String`: The stripped endpoint URL (trailing slashes removed).
|
|
"""
|
|
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
|
return strip(aws.endpoint, '/')
|
|
end
|
|
|
|
""" Generate the full service URL including the resource path for a Garage S3 request.
|
|
|
|
# Arguments
|
|
- `aws::GarageConfig`
|
|
The Garage configuration containing the endpoint.
|
|
- `service::String`
|
|
The AWS service name (e.g., `"s3"`).
|
|
- `resource::String`
|
|
The S3 resource path (e.g., `"/bucket/key"`).
|
|
|
|
# Return
|
|
- `String`: The full URL combining the endpoint and resource path.
|
|
"""
|
|
function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
|
|
endpoint = strip(aws.endpoint, '/')
|
|
resource_path = startswith(resource, '/') ? resource : "/" * resource
|
|
return string(endpoint, resource_path)
|
|
end
|
|
|
|
# ===================================================================
|
|
# 2. Thread-Safe Storage Client Wrapper
|
|
# ===================================================================
|
|
|
|
""" Create a thread-safe Garage S3 storage client.
|
|
|
|
# Arguments
|
|
- `endpoint::String`
|
|
The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
|
|
- `key::String`
|
|
The Garage access key ID (created via garage-ui).
|
|
- `secret::String`
|
|
The secret key corresponding to the access key.
|
|
- `bucket::String`
|
|
The name of the S3 bucket to use.
|
|
|
|
# Keyword Arguments
|
|
- `region::String = "garage"`
|
|
The region identifier for the Garage instance.
|
|
|
|
# Return
|
|
- `GarageStorage`: A thread-safe storage client instance.
|
|
|
|
# Examples
|
|
```jldoctest
|
|
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
|
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
|
```
|
|
"""
|
|
struct GarageStorage
|
|
config::GarageConfig
|
|
bucket::String
|
|
end
|
|
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
|
creds = SimpleCredentials(key, secret, "")
|
|
config = GarageConfig(endpoint, region, creds)
|
|
return GarageStorage(config, bucket)
|
|
end
|
|
|
|
""" Upload an object to the Garage S3 bucket (callable struct).
|
|
|
|
# Arguments
|
|
- `key::String`
|
|
The object key (name) in the bucket. Must not contain slashes (`/`).
|
|
- `data::Union{String, Vector{UInt8}}`
|
|
The data to upload.
|
|
|
|
# Return
|
|
- `String`: The file download URL if the upload succeeds.
|
|
- `nothing` if the key contains slashes (upload aborted with warning).
|
|
|
|
# Notes
|
|
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
|
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
|
|
|
# Examples
|
|
```jldoctest
|
|
julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
|
julia> put_file1 = put_file(storage1)
|
|
julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}")
|
|
Successfully uploaded: test.json
|
|
"https://s3-api.yiem.cc/sommpanion-s3/test.json"
|
|
```
|
|
"""
|
|
struct put_file
|
|
storage::GarageStorage
|
|
end
|
|
function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}})::String
|
|
# Check if the key contains a slash (virtual folder character)
|
|
if occursin('/', key)
|
|
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
|
return nothing
|
|
end
|
|
|
|
# Proceed if the key is flat
|
|
s3_put(pf.storage.config, pf.storage.bucket, key, data)
|
|
println("Successfully uploaded: ", key)
|
|
|
|
# file download URL
|
|
objURL = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
|
|
|
|
return objURL
|
|
end
|
|
|
|
""" Upload an object to the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
- `key::String`
|
|
The object key (name) in the bucket. Must not contain slashes (`/`).
|
|
- `data::Union{String, Vector{UInt8}}`
|
|
The data to upload.
|
|
|
|
# Return
|
|
- `Nothing` if the key contains slashes (upload aborted with warning).
|
|
- Prints a success message if the upload completes.
|
|
|
|
# Notes
|
|
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
|
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
|
|
|
# Examples
|
|
```julia
|
|
julia> put_file(storage, "test.json", "{\"status\": \"active\"}")
|
|
Successfully uploaded: test.json
|
|
```
|
|
"""
|
|
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})::String
|
|
# Check if the key contains a slash (virtual folder character)
|
|
if occursin('/', key)
|
|
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
|
return nothing
|
|
end
|
|
|
|
# Proceed if the key is flat
|
|
s3_put(storage.config, storage.bucket, key, data)
|
|
println("Successfully uploaded: ", key)
|
|
|
|
# file download URL
|
|
objURL = "$(storage.config.endpoint)/$(storage.bucket)/$key"
|
|
|
|
return objURL
|
|
end
|
|
|
|
""" Download an object from the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
- `key::String`
|
|
The object key (name) in the bucket.
|
|
|
|
# Return
|
|
- `Vector{UInt8}`: The raw bytes of the downloaded object.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> data = String(get_file(storage, "test.json"))
|
|
"\"{\\\"status\\\": \\\"active\\\"}\""
|
|
```
|
|
"""
|
|
function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
|
|
return s3_get(storage.config, storage.bucket, key)
|
|
end
|
|
|
|
""" List all object keys in the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
|
|
# Return
|
|
- `Vector{String}`: A list of object keys (file names) in the bucket.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> keys = list_files(storage)
|
|
3-element Vector{String}:
|
|
"test1.json"
|
|
"test2.json"
|
|
"users-1002.json"
|
|
```
|
|
"""
|
|
function list_files(storage::GarageStorage)
|
|
# Approach 2: s3_list_objects returns a Vector of Dicts with object details
|
|
objects = s3_list_objects(storage.config, storage.bucket)
|
|
return [obj["Key"] for obj in objects]
|
|
end
|
|
|
|
""" Delete an object from the Garage S3 bucket.
|
|
|
|
# Arguments
|
|
- `storage::GarageStorage`
|
|
The storage client instance.
|
|
- `key::String`
|
|
The object key (name) to delete.
|
|
|
|
# Examples
|
|
```julia
|
|
julia> delete_file(storage, "test.json")
|
|
```
|
|
"""
|
|
function delete_file(storage::GarageStorage, key::String)
|
|
s3_delete(storage.config, storage.bucket, key)
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
end # module GarageS3
|