1st commit
This commit is contained in:
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
name = "GeneralUtils"
|
name = "GeneralUtils"
|
||||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||||
version = "0.6.1"
|
version = "0.6.2"
|
||||||
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
||||||
|
|
||||||
[deps]
|
[deps]
|
||||||
|
|||||||
@@ -1,271 +0,0 @@
|
|||||||
using SHA
|
|
||||||
using Dates
|
|
||||||
using URIs
|
|
||||||
using OrderedCollections
|
|
||||||
using AWS
|
|
||||||
@service S3
|
|
||||||
using AWSS3
|
|
||||||
|
|
||||||
|
|
||||||
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
|
||||||
# this file use local AWS config
|
|
||||||
|
|
||||||
""" Example
|
|
||||||
storage = GeneralUtils.GarageStorage(
|
|
||||||
"https://s3-api.yiem.cc",
|
|
||||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
|
||||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
|
||||||
"sommpanion-s3"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
|
||||||
GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
|
|
||||||
keys = GeneralUtils.list_files(storage)
|
|
||||||
|
|
||||||
println("Read back: ", data)
|
|
||||||
println("Bucket keys: ", keys)
|
|
||||||
|
|
||||||
# test with curl
|
|
||||||
curl -v \
|
|
||||||
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
|
||||||
http://192.168.88.106:3902/users-1002.json
|
|
||||||
|
|
||||||
# test with API
|
|
||||||
data = String(GeneralUtils.get_file(storage, "users-1005.json"))
|
|
||||||
|
|
||||||
# test with a browser
|
|
||||||
https://s3-web.mydomain.com/my_bucket_name/users-1005.json
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# ===================================================================
|
|
||||||
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
|
||||||
# ===================================================================
|
|
||||||
|
|
||||||
struct SimpleCredentials
|
|
||||||
access_key_id::String
|
|
||||||
secret_key::String
|
|
||||||
token::String
|
|
||||||
end
|
|
||||||
|
|
||||||
struct GarageConfig <: AWS.AbstractAWSConfig
|
|
||||||
endpoint::String
|
|
||||||
region::String
|
|
||||||
credentials::SimpleCredentials
|
|
||||||
end
|
|
||||||
|
|
||||||
# Required extensions for AWS.jl pipeline
|
|
||||||
AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
|
|
||||||
AWS.credentials(c::SimpleCredentials) = c
|
|
||||||
AWS.check_credentials(c::SimpleCredentials) = c
|
|
||||||
AWS.region(aws::GarageConfig) = aws.region
|
|
||||||
AWS.credentials(aws::GarageConfig) = aws.credentials
|
|
||||||
|
|
||||||
""" Generate the base URL for a Garage S3 service request.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `aws::GarageConfig`
|
|
||||||
The Garage configuration containing the endpoint.
|
|
||||||
- `service::String`
|
|
||||||
The AWS service name (e.g., `"s3"`).
|
|
||||||
- `region::String`
|
|
||||||
The AWS region string.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `String`: The stripped endpoint URL (trailing slashes removed).
|
|
||||||
"""
|
|
||||||
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
|
||||||
return strip(aws.endpoint, '/')
|
|
||||||
end
|
|
||||||
|
|
||||||
""" Generate the full service URL including the resource path for a Garage S3 request.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `aws::GarageConfig`
|
|
||||||
The Garage configuration containing the endpoint.
|
|
||||||
- `service::String`
|
|
||||||
The AWS service name (e.g., `"s3"`).
|
|
||||||
- `resource::String`
|
|
||||||
The S3 resource path (e.g., `"/bucket/key"`).
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `String`: The full URL combining the endpoint and resource path.
|
|
||||||
"""
|
|
||||||
function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
|
|
||||||
endpoint = strip(aws.endpoint, '/')
|
|
||||||
resource_path = startswith(resource, '/') ? resource : "/" * resource
|
|
||||||
return string(endpoint, resource_path)
|
|
||||||
end
|
|
||||||
|
|
||||||
# ===================================================================
|
|
||||||
# 2. Thread-Safe Storage Client Wrapper
|
|
||||||
# ===================================================================
|
|
||||||
|
|
||||||
""" Create a thread-safe Garage S3 storage client.
|
|
||||||
|
|
||||||
# Arguments
|
|
||||||
- `endpoint::String`
|
|
||||||
The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
|
|
||||||
- `key::String`
|
|
||||||
The Garage access key ID (created via garage-ui).
|
|
||||||
- `secret::String`
|
|
||||||
The secret key corresponding to the access key.
|
|
||||||
- `bucket::String`
|
|
||||||
The name of the S3 bucket to use.
|
|
||||||
|
|
||||||
# Keyword Arguments
|
|
||||||
- `region::String = "garage"`
|
|
||||||
The region identifier for the Garage instance.
|
|
||||||
|
|
||||||
# Return
|
|
||||||
- `GarageStorage`: A thread-safe storage client instance.
|
|
||||||
|
|
||||||
# Examples
|
|
||||||
```jldoctest
|
|
||||||
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
|
||||||
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
struct GarageStorage
|
|
||||||
config::GarageConfig
|
|
||||||
bucket::String
|
|
||||||
end
|
|
||||||
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
|
||||||
creds = SimpleCredentials(key, secret, "")
|
|
||||||
config = GarageConfig(endpoint, region, creds)
|
|
||||||
return GarageStorage(config, bucket)
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl
|
|
||||||
function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}})
|
|
||||||
return SHA.hmac_sha256(key, msg)
|
|
||||||
end
|
|
||||||
|
|
||||||
function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}})
|
|
||||||
return SHA.hmac_sha256(Vector{UInt8}(key), msg)
|
|
||||||
end
|
|
||||||
|
|
||||||
# 2. AWSS3 overload targeting your custom Garage endpoint
|
|
||||||
function AWSS3._s3_sign_url_v4(
|
|
||||||
aws::GarageConfig,
|
|
||||||
bucket::String,
|
|
||||||
path::String,
|
|
||||||
seconds::Int=3600;
|
|
||||||
verb::String="GET",
|
|
||||||
content_type::String="application/octet-stream",
|
|
||||||
protocol::String="https",
|
|
||||||
)
|
|
||||||
path_escaped = URIs.escapepath("/$bucket/$path")
|
|
||||||
|
|
||||||
now_datetime = now(Dates.UTC)
|
|
||||||
datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
|
|
||||||
date_stamp = Dates.format(now_datetime, "YYYYmmdd")
|
|
||||||
|
|
||||||
service = "s3"
|
|
||||||
scheme = "AWS4"
|
|
||||||
algorithm = "HMAC-SHA256"
|
|
||||||
terminator = "aws4_request"
|
|
||||||
|
|
||||||
# Robust host & port extraction (prevents trailing colon bugs like 'domain.com:')
|
|
||||||
parsed_uri = URIs.URI(aws.endpoint)
|
|
||||||
has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0
|
|
||||||
host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host
|
|
||||||
effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme
|
|
||||||
|
|
||||||
scope = "$date_stamp/$(aws.region)/$service/$terminator"
|
|
||||||
|
|
||||||
headers = OrderedDict{String,String}("Host" => host)
|
|
||||||
sort!(headers; by=name -> lowercase(name))
|
|
||||||
canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
|
|
||||||
|
|
||||||
query = OrderedDict{String,String}(
|
|
||||||
"X-Amz-Expires" => string(seconds),
|
|
||||||
"X-Amz-Algorithm" => "$scheme-$algorithm",
|
|
||||||
"X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope",
|
|
||||||
"X-Amz-Date" => datetime_stamp,
|
|
||||||
"X-Amz-SignedHeaders" => canonical_header_names,
|
|
||||||
)
|
|
||||||
|
|
||||||
if !isempty(aws.credentials.token)
|
|
||||||
query["X-Amz-Security-Token"] = aws.credentials.token
|
|
||||||
end
|
|
||||||
|
|
||||||
sort!(query; by=name -> lowercase(name))
|
|
||||||
|
|
||||||
canonical_headers = join(
|
|
||||||
map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers))
|
|
||||||
)
|
|
||||||
|
|
||||||
canonical_request = string(
|
|
||||||
"$verb\n",
|
|
||||||
"$path_escaped\n",
|
|
||||||
"$(URIs.escapeuri(query))\n",
|
|
||||||
"$canonical_headers\n",
|
|
||||||
"$canonical_header_names\n",
|
|
||||||
"UNSIGNED-PAYLOAD",
|
|
||||||
)
|
|
||||||
|
|
||||||
hashed_canonical_request = bytes2hex(sha256(canonical_request))
|
|
||||||
|
|
||||||
string_to_sign = string(
|
|
||||||
"$scheme-$algorithm\n",
|
|
||||||
"$datetime_stamp\n",
|
|
||||||
"$scope\n",
|
|
||||||
hashed_canonical_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
# AWS Signature V4 Key Derivation Chain
|
|
||||||
key_secret = string(scheme, aws.credentials.secret_key)
|
|
||||||
key_date = _hmac_sha256(key_secret, date_stamp)
|
|
||||||
key_region = _hmac_sha256(key_date, aws.region)
|
|
||||||
key_service = _hmac_sha256(key_region, service)
|
|
||||||
key_signing = _hmac_sha256(key_service, terminator)
|
|
||||||
signature = _hmac_sha256(key_signing, string_to_sign)
|
|
||||||
|
|
||||||
query["X-Amz-Signature"] = bytes2hex(signature)
|
|
||||||
|
|
||||||
return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query))
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
|
|
||||||
|
|
||||||
Generates direct public HTTP GET URLs via Garage's s3-web interface.
|
|
||||||
Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint.
|
|
||||||
"""
|
|
||||||
function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com")
|
|
||||||
clean_endpoint = strip(web_endpoint, '/')
|
|
||||||
clean_key = lstrip(key, '/')
|
|
||||||
|
|
||||||
# s3-web URL format: https://<web-domain>/<bucket>/<key>
|
|
||||||
download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)"
|
|
||||||
|
|
||||||
return (
|
|
||||||
download_url = download_url,
|
|
||||||
# DELETE requires s3-api (s3-web does not support HTTP DELETE)
|
|
||||||
delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE")
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# 1. Initialize Garage configuration pointing to S3 API domain
|
|
||||||
storage = GarageStorage(
|
|
||||||
"https://s3-api.yiem.cc",
|
|
||||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
|
||||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
|
||||||
"sommpanion-s3"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 2. Upload microservice file payload
|
|
||||||
put_file(storage, "job-9042-data.bin", payload_bytes)
|
|
||||||
|
|
||||||
# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes)
|
|
||||||
urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900)
|
|
||||||
|
|
||||||
# Pass `urls.download_url` and `urls.delete_url` to the downstream service
|
|
||||||
Reference in New Issue
Block a user