From 3fb7457f619973cb81769a588f6f9dec0ff147a1 Mon Sep 17 00:00:00 2001 From: narawat Date: Sat, 29 Aug 2026 08:22:46 +0700 Subject: [PATCH] 1st commit --- Project.toml | 2 +- etc.jl | 271 --------------------------------------------------- main.jl | 2 + 3 files changed, 3 insertions(+), 272 deletions(-) delete mode 100644 etc.jl create mode 100644 main.jl diff --git a/Project.toml b/Project.toml index 829d4a0..d798a3c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -version = "0.6.1" +version = "0.6.2" authors = ["tonaerospace "] [deps] diff --git a/etc.jl b/etc.jl deleted file mode 100644 index 70bf716..0000000 --- a/etc.jl +++ /dev/null @@ -1,271 +0,0 @@ -using SHA -using Dates -using URIs -using OrderedCollections -using AWS -@service S3 -using AWSS3 - - -# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key). -# this file use local AWS config - -""" Example -storage = GeneralUtils.GarageStorage( - "https://s3-api.yiem.cc", - "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) - "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "sommpanion-s3" -) - -# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) -GeneralUtils.put_file(storage, "users-1005.json", "{\"status\": \"active\"}") -keys = GeneralUtils.list_files(storage) - -println("Read back: ", data) -println("Bucket keys: ", keys) - -# test with curl -curl -v \ - -H 'Host: sommpanion-s3.s3-web.yiem.cc' \ - http://192.168.88.106:3902/users-1002.json - -# test with API -data = String(GeneralUtils.get_file(storage, "users-1005.json")) - -# test with a browser -https://s3-web.mydomain.com/my_bucket_name/users-1005.json - -""" - -# =================================================================== -# 1. Custom GarageConfig Definition (Thread-Safe, No Global State) -# =================================================================== - -struct SimpleCredentials - access_key_id::String - secret_key::String - token::String -end - -struct GarageConfig <: AWS.AbstractAWSConfig - endpoint::String - region::String - credentials::SimpleCredentials -end - -# Required extensions for AWS.jl pipeline -AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c -AWS.credentials(c::SimpleCredentials) = c -AWS.check_credentials(c::SimpleCredentials) = c -AWS.region(aws::GarageConfig) = aws.region -AWS.credentials(aws::GarageConfig) = aws.credentials - -""" Generate the base URL for a Garage S3 service request. - -# Arguments -- `aws::GarageConfig` - The Garage configuration containing the endpoint. -- `service::String` - The AWS service name (e.g., `"s3"`). -- `region::String` - The AWS region string. - -# Return -- `String`: The stripped endpoint URL (trailing slashes removed). -""" -function AWS.generate_service_url(aws::GarageConfig, service::String, region::String) - return strip(aws.endpoint, '/') -end - -""" Generate the full service URL including the resource path for a Garage S3 request. - -# Arguments -- `aws::GarageConfig` - The Garage configuration containing the endpoint. -- `service::String` - The AWS service name (e.g., `"s3"`). -- `resource::String` - The S3 resource path (e.g., `"/bucket/key"`). - -# Return -- `String`: The full URL combining the endpoint and resource path. -""" -function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String) - endpoint = strip(aws.endpoint, '/') - resource_path = startswith(resource, '/') ? resource : "/" * resource - return string(endpoint, resource_path) -end - -# =================================================================== -# 2. Thread-Safe Storage Client Wrapper -# =================================================================== - -""" Create a thread-safe Garage S3 storage client. - -# Arguments -- `endpoint::String` - The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`). -- `key::String` - The Garage access key ID (created via garage-ui). -- `secret::String` - The secret key corresponding to the access key. -- `bucket::String` - The name of the S3 bucket to use. - -# Keyword Arguments -- `region::String = "garage"` - The region identifier for the Garage instance. - -# Return -- `GarageStorage`: A thread-safe storage client instance. - -# Examples -```jldoctest -julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") -GarageStorage(GarageConfig(...), "sommpanion-s3") -``` -""" -struct GarageStorage - config::GarageConfig - bucket::String -end -function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage") - creds = SimpleCredentials(key, secret, "") - config = GarageConfig(endpoint, region, creds) - return GarageStorage(config, bucket) -end - - -# 1. Helper for AWS SigV4 HMAC-SHA256 calculations using SHA.jl -function _hmac_sha256(key::Vector{UInt8}, msg::Union{String, Vector{UInt8}}) - return SHA.hmac_sha256(key, msg) -end - -function _hmac_sha256(key::String, msg::Union{String, Vector{UInt8}}) - return SHA.hmac_sha256(Vector{UInt8}(key), msg) -end - -# 2. AWSS3 overload targeting your custom Garage endpoint -function AWSS3._s3_sign_url_v4( - aws::GarageConfig, - bucket::String, - path::String, - seconds::Int=3600; - verb::String="GET", - content_type::String="application/octet-stream", - protocol::String="https", -) - path_escaped = URIs.escapepath("/$bucket/$path") - - now_datetime = now(Dates.UTC) - datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z") - date_stamp = Dates.format(now_datetime, "YYYYmmdd") - - service = "s3" - scheme = "AWS4" - algorithm = "HMAC-SHA256" - terminator = "aws4_request" - - # Robust host & port extraction (prevents trailing colon bugs like 'domain.com:') - parsed_uri = URIs.URI(aws.endpoint) - has_port = !isnothing(parsed_uri.port) && parsed_uri.port != 0 - host = has_port ? "$(parsed_uri.host):$(parsed_uri.port)" : parsed_uri.host - effective_protocol = isempty(parsed_uri.scheme) ? protocol : parsed_uri.scheme - - scope = "$date_stamp/$(aws.region)/$service/$terminator" - - headers = OrderedDict{String,String}("Host" => host) - sort!(headers; by=name -> lowercase(name)) - canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") - - query = OrderedDict{String,String}( - "X-Amz-Expires" => string(seconds), - "X-Amz-Algorithm" => "$scheme-$algorithm", - "X-Amz-Credential" => "$(aws.credentials.access_key_id)/$scope", - "X-Amz-Date" => datetime_stamp, - "X-Amz-SignedHeaders" => canonical_header_names, - ) - - if !isempty(aws.credentials.token) - query["X-Amz-Security-Token"] = aws.credentials.token - end - - sort!(query; by=name -> lowercase(name)) - - canonical_headers = join( - map(header -> "$(lowercase(header.first)):$(lowercase(header.second))\n", collect(headers)) - ) - - canonical_request = string( - "$verb\n", - "$path_escaped\n", - "$(URIs.escapeuri(query))\n", - "$canonical_headers\n", - "$canonical_header_names\n", - "UNSIGNED-PAYLOAD", - ) - - hashed_canonical_request = bytes2hex(sha256(canonical_request)) - - string_to_sign = string( - "$scheme-$algorithm\n", - "$datetime_stamp\n", - "$scope\n", - hashed_canonical_request, - ) - - # AWS Signature V4 Key Derivation Chain - key_secret = string(scheme, aws.credentials.secret_key) - key_date = _hmac_sha256(key_secret, date_stamp) - key_region = _hmac_sha256(key_date, aws.region) - key_service = _hmac_sha256(key_region, service) - key_signing = _hmac_sha256(key_service, terminator) - signature = _hmac_sha256(key_signing, string_to_sign) - - query["X-Amz-Signature"] = bytes2hex(signature) - - return string(effective_protocol, "://", host, path_escaped, "?", URIs.escapeuri(query)) -end - - - -""" - generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") - -Generates direct public HTTP GET URLs via Garage's s3-web interface. -Note: DELETE is not supported by s3-web and must fall back to the s3-api endpoint. -""" -function generate_web_urls(storage::GarageStorage, key::String; web_endpoint::String="https://s3-web.mydomain.com") - clean_endpoint = strip(web_endpoint, '/') - clean_key = lstrip(key, '/') - - # s3-web URL format: https://// - download_url = "$(clean_endpoint)/$(storage.bucket)/$(clean_key)" - - return ( - download_url = download_url, - # DELETE requires s3-api (s3-web does not support HTTP DELETE) - delete_url = AWSS3.s3_sign_url(storage.config, storage.bucket, key, 900; verb="DELETE") - ) -end - - - - - -# 1. Initialize Garage configuration pointing to S3 API domain -storage = GarageStorage( - "https://s3-api.yiem.cc", - "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) - "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "sommpanion-s3" -) - -# 2. Upload microservice file payload -put_file(storage, "job-9042-data.bin", payload_bytes) - -# 3. Generate pre-signed ephemeral URLs (valid for 15 minutes) -urls = generate_ephemeral_urls(storage, "info"; ttl_seconds=900) - -# Pass `urls.download_url` and `urls.delete_url` to the downstream service \ No newline at end of file diff --git a/main.jl b/main.jl new file mode 100644 index 0000000..95cc800 --- /dev/null +++ b/main.jl @@ -0,0 +1,2 @@ +# dummy main.jl so i can do GeneralUtils precompile +using GeneralUtils \ No newline at end of file