diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..afa2dd6 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,27 @@ +# 1. Garage Web Admin UI +garage-ui.yiem.cc { + reverse_proxy 192.168.88.106:3909 +} + +# 2. Garage S3 API (Path-Style & Single Domain) +s3-api.yiem.cc { + reverse_proxy 192.168.88.106:3900 { + header_up Host {host} + } +} + +# 3. Garage S3 WEB (Path-Style & Single Domain) +s3-web.yiem.cc { + # /bucket -> /bucket/ + @bucket_root path_regexp bucket_root ^/([^/]+)$ + redir @bucket_root /{http.regexp.bucket_root.1}/ 301 + + # /bucket/... -> Garage + @bucket path_regexp bucket ^/([^/]+)(/.*)?$ + + rewrite @bucket {http.regexp.bucket.2} + + reverse_proxy 192.168.88.106:3902 { + header_up Host {http.regexp.bucket.1}.s3-web.yiem.cc + } +} diff --git a/garage.toml b/garage.toml new file mode 100644 index 0000000..e9bcdd5 --- /dev/null +++ b/garage.toml @@ -0,0 +1,52 @@ +# ========================================== +# Garage Storage Engine Configuration +# ========================================== + +# Core storage directories +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" + +# LMDB is recommended for production performance +db_engine = "lmdb" +metadata_auto_snapshot_interval = "6h" + +# Multi-node ready setup: +# Set to 1 for initial single node. Update to 3 when adding Node 2 and Node 3 later. +replication_factor = 1 +compression_level = 2 + +# ========================================== +# Network & Node Identity Configuration +# ========================================== + +# Bind to all interfaces for host network mode +rpc_bind_addr = "[::]:3901" + +# IMPORTANT: Set to Node 1's publicly reachable IP or DNS name (e.g., 192.168.1.10:3901 or 203.0.113.10:3901) +rpc_public_addr = "192.168.88.106:3901" + +# Generate with: openssl rand -hex 32 (MUST BE IDENTICAL ON ALL NODES IN CLUSTER) +rpc_secret = "3cf7a4d1da5d7b024a06db7fe6d73e2c160c6de640fda69b0b964d98678155a7" + +# ========================================== +# Service APIs +# ========================================== + +[s3_api] +s3_region = "garage" +api_bind_addr = "0.0.0.0:3900" +root_domain = ".s3-api.yiem.cc" + +[s3_web] +bind_addr = "0.0.0.0:3902" +root_domain = ".s3-web.yiem.cc" +index = "index.html" + +[k2v_api] +api_bind_addr = "0.0.0.0:3904" + +[admin] +api_bind_addr = "0.0.0.0:3903" +# Generate with: openssl rand -hex 16 +admin_token = "fd92f47a7c1ce8b072b06bb19dbc3fef" +metrics_token = "27d29618cc436a23ec7d842392fdf74b" \ No newline at end of file diff --git a/garage_docker-compose.yml b/garage_docker-compose.yml new file mode 100644 index 0000000..bc88efc --- /dev/null +++ b/garage_docker-compose.yml @@ -0,0 +1,48 @@ +services: + garage: + image: dxflrs/garage:v2.3.0 + container_name: garage + restart: unless-stopped + network_mode: "host" + volumes: + - ./garage.toml:/etc/garage.toml:ro + - garage_meta:/var/lib/garage/meta + - garage_data:/var/lib/garage/data + environment: + - RUST_LOG=garage=trace + healthcheck: + test: ["CMD", "/garage", "status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + + webui: + image: khairul169/garage-webui:latest + container_name: garage-webui + restart: unless-stopped + network_mode: "host" + volumes: + - ./garage.toml:/etc/garage.toml:ro + environment: + API_BASE_URL: "http://127.0.0.1:3903" + S3_ENDPOINT_URL: "https://s3-api.yiem.cc" + # Format: "username:bcrypt_hash" (generate via: htpasswd -nbBC 10 "admin" "yourpassword") + AUTH_USER_PASS: "admin:$$2y$$10$$fhEucJHW8N5p8NL7Gv4Bi.FZVngbq/zatVDCjl.V.xNtj5q1PNUZW" + depends_on: + garage: + condition: service_healthy + +volumes: + garage_meta: + driver: local + driver_opts: + type: none + o: bind + device: ./data/meta + garage_data: + driver: local + driver_opts: + type: none + o: bind + device: ./data/storage \ No newline at end of file diff --git a/photo.jpg b/photo.jpg new file mode 100644 index 0000000..7de5a42 Binary files /dev/null and b/photo.jpg differ diff --git a/src/garageS3.jl b/src/garageS3.jl index a00f490..69f7fde 100644 --- a/src/garageS3.jl +++ b/src/garageS3.jl @@ -4,7 +4,7 @@ export LifecycleExpiration, build_lifecycle_xml, GarageStorage, put_file, get_file, list_files, delete_file, set_lifecycle_expiration, get_lifecycle, delete_lifecycle, - s3_sign_url + s3_sign_url, s3_sign_url_web using AWS @service S3 @@ -24,7 +24,7 @@ storage1 = GeneralUtils.GarageStorage( "https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc) "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key - "testbucket" + "sommpanion" ) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) @@ -347,6 +347,9 @@ function s3_sign_url( scope = "$date_stamp/$region/$service/$terminator" headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host) + if verb == "PUT" + headers["Content-Type"] = content_type + end sort!(headers; by=name -> lowercase(name)) canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";") @@ -355,11 +358,11 @@ function s3_sign_url( "X-Amz-Algorithm" => "$scheme-$algorithm", "X-Amz-Credential" => "$(creds.access_key_id)/$scope", "X-Amz-Date" => datetime_stamp, - "X-Amz-Security-Token" => isempty(creds.token) ? "" : creds.token, "X-Amz-SignedHeaders" => canonical_header_names, ) - - sort!(query; by=name -> lowercase(name)) + if !isempty(creds.token) + query["X-Amz-Security-Token"] = creds.token + end canonical_headers = join( map( @@ -370,12 +373,24 @@ function s3_sign_url( resource_path = "/$bucket/$key" + # Build canonical query string: sort by key=value pair lexicographically, + # URL-encoding each key and value (matching Garage's uri_encode) + sorted_q = sort( + [URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)], + by=lowercase, + ) + canonical_query = join(sorted_q, "&") + canonical_request = string( "$verb\n", - "$(URIs.escapeuri(resource_path))\n", - "$(URIs.escapeuri(query))\n", - "$canonical_headers\n", - "$canonical_header_names\n", + resource_path, + "\n", + canonical_query, + "\n", + canonical_headers, + "\n", + canonical_header_names, + "\n", "UNSIGNED-PAYLOAD", ) @@ -386,8 +401,8 @@ function s3_sign_url( bytes2hex(digest(MD_SHA256, canonical_request)), ) - key_secret = string(scheme, creds.secret_key) - key_date = digest(MD_SHA256, date_stamp, key_secret) + key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key)) + key_date = digest(MD_SHA256, date_stamp, key_secret_bytes) key_region = digest(MD_SHA256, region, key_date) key_service = digest(MD_SHA256, service, key_region) key_signing = digest(MD_SHA256, terminator, key_service) @@ -399,6 +414,31 @@ function s3_sign_url( return "$(endpoint)/$bucket/$key?$query_string" end +""" Generate a pre-signed URL using the Garage S3 web endpoint (s3-web). + +Useful for direct browser access — URLs look like +`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...` + +Arguments and keyword arguments are the same as `s3_sign_url`. +""" +function s3_sign_url_web( + storage::GarageStorage, + key::String, + seconds::Int=3600; + verb::String="GET", + content_type::String="application/octet-stream", + signature_version::String="v4", +)::String + web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web") + web_creds = SimpleCredentials( + storage.config.credentials.access_key_id, + storage.config.credentials.secret_key, + storage.config.credentials.token, + ) + web_config = GarageConfig(web_endpoint, storage.config.region, web_creds) + web_storage = GarageStorage(web_config, storage.bucket) + return s3_sign_url(web_storage, key, seconds; verb, content_type, signature_version) +end # =================================================================== # 3. Bucket Lifecycle Configuration diff --git a/test/etc.jl b/test/etc.jl index e69de29..a3bc90e 100644 --- a/test/etc.jl +++ b/test/etc.jl @@ -0,0 +1,130 @@ +curl -X PUT -H "Content-Type: application/json" \ + -d '{"test": "data"}' \ + "https://s3-api.yiem.cc/sommpanion/pcimage.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T103745Z&X-Amz-Expires=300&X-Amz-Security-Token=&X-Amz-SignedHeaders=host&X-Amz-Signature=0e195a48f824a2f8ffcb06461396cd6a8bddd5b61d93e9006f947ae10ecf525f" + + + + + + + + + +curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ + -H 'Host: s3-api.yiem.cc' \ + -H 'Content-Type: image/jpeg' \ + "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=3600&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T025455Z&X-Amz-SignedHeaders=host&X-Amz-Signature=92e86746a92fa07914d8f08169dac66e2c64f36b807f86504bf8efec41fed153" + + + + + + + + + + + + + + + + +read +./garage.toml +./Caddyfile +./src/garageS3.jl + + + +julia> url = GeneralUtils.s3_sign_url(storage1, "photo.jpg", 36000; verb="PUT", content_type="image/jpeg") +"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T145004Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=92cd634dbb0f66db4b8b039ef5939c138203de260e8342fe27ef78979d90a824" + +--- + + + + + + + + + + + +ton@server12vm-dev:~/docker-apps/sommpanion/etc/AWS.jl$ +curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ + -H 'Host: s3-api.yiem.cc' \ + -H 'Content-Type: image/jpeg' \ + "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02" + +Note: Unnecessary use of -X or --request, PUT is already inferred. +* Trying 110.77.160.154:443... +* Connected to s3-api.yiem.cc (110.77.160.154) port 443 (#0) +* ALPN: offers h2,http/1.1 +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +* CAfile: /etc/ssl/certs/ca-certificates.crt +* CApath: /etc/ssl/certs +* TLSv1.3 (IN), TLS handshake, Server hello (2): +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +* TLSv1.3 (IN), TLS handshake, Certificate (11): +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +* TLSv1.3 (IN), TLS handshake, Finished (20): +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +* TLSv1.3 (OUT), TLS handshake, Finished (20): +* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256 +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=s3-api.yiem.cc +* start date: Aug 27 04:27:02 2026 GMT +* expire date: Nov 25 04:27:01 2026 GMT +* subjectAltName: host "s3-api.yiem.cc" matched cert's "s3-api.yiem.cc" +* issuer: C=US; O=Let's Encrypt; CN=YE2 +* SSL certificate verify ok. +* using HTTP/2 +* h2h3 [:method: PUT] +* h2h3 [:path: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02] +* h2h3 [:scheme: https] +* h2h3 [:authority: s3-api.yiem.cc] +* h2h3 [user-agent: curl/7.88.1] +* h2h3 [accept: */*] +* h2h3 [content-type: image/jpeg] +* h2h3 [content-length: 439490] +* Using Stream ID: 1 (easy handle 0x559a08824c80) +> PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 HTTP/2 +> Host: s3-api.yiem.cc +> user-agent: curl/7.88.1 +> accept: */* +> content-type: image/jpeg +> content-length: 439490 +> +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +* We are completely uploaded and fine +< HTTP/2 403 +< access-control-allow-origin: * +< alt-svc: h3=":443"; ma=2592000 +< content-type: application/xml +< date: Fri, 04 Sep 2026 15:23:13 GMT +< via: 1.1 Caddy +< content-length: 190 +< +* Connection #0 to host s3-api.yiem.cc left intact +AccessDeniedForbidden: Invalid signature/sommpanion/photo.jpggarage + +2026-09-04T15:23:13.796037Z INFO garage_api_common::generic_server: 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 +2026-09-04T15:23:13.796077Z DEBUG garage_api_common::generic_server: Request { method: PUT, uri: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02, version: HTTP/1.1, headers: {"host": "s3-api.yiem.cc", "user-agent": "curl/7.88.1", "content-length": "439490", "accept": "*/*", "content-type": "image/jpeg", "via": "2.0 Caddy", "x-forwarded-for": "110.77.160.154", "x-forwarded-host": "s3-api.yiem.cc", "x-forwarded-proto": "https", "accept-encoding": "gzip"}, body: Body(Streaming) } +2026-09-04T15:23:13.796181Z DEBUG garage_api_common::generic_server: Endpoint: PutObject +2026-09-04T15:23:13.796461Z INFO garage_api_common::generic_server: error 403 Forbidden, Forbidden: Invalid signature in response to 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 + + + + + + + + + + + + + diff --git a/test/runtests.jl b/test/runtests.jl index 6f0c92f..e19cd29 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,8 +2,6 @@ using Test using GeneralUtils using GeneralUtils.garageS3 using AWS -using Random: randstring -using UUIDs: uuid4 using URIs @testset "garageS3.jl" begin @@ -143,7 +141,7 @@ using URIs "testbucket" ) - test_key = "testrunner-$(randstring(8)).json" + test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}" # Test put_file(storage, key, data) - successful upload @@ -166,7 +164,7 @@ using URIs @test test_key in keys # Test put_file callable struct - test_key2 = "testrunner-callable-$(randstring(8)).json" + test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json" callable_result = put_file(storage)(test_key2, "callable-data") @test callable_result !== nothing @test callable_result.api != "" @@ -195,7 +193,7 @@ using URIs "testbucket" ) - test_key = "single/slash/test-$(randstring(8)).json" + test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"level\": 1, \"nested\": true}" # Upload nested key @@ -228,7 +226,7 @@ using URIs "testbucket" ) - test_key = "deep/nested/folder/subfolder/test-$(randstring(8)).json" + test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}" # Upload deeply nested key @@ -260,7 +258,7 @@ using URIs "testbucket" ) - test_key = "callable/nested/test-$(randstring(8)).json" + test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json" test_data = "{\"callable\": true, \"nested\": true}" uploader = put_file(storage) @@ -282,7 +280,7 @@ using URIs "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "testbucket" ) - result = get_file(storage, "nonexistent-key-$(randstring(8)).json") + result = get_file(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json") @test result === nothing end @@ -294,7 +292,7 @@ using URIs "testbucket" ) - test_key = "testrunner-binary-$(randstring(8)).bin" + test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin" binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f] result = put_file(storage, test_key, binary_data) @@ -315,7 +313,7 @@ using URIs "testbucket" ) - test_key = "testrunner-list-$(randstring(8)).txt" + test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt" put_file(storage, test_key, "list-test-data") keys = list_files(storage) @@ -333,7 +331,7 @@ using URIs "testbucket" ) - test_key = "testrunner-url-$(randstring(8)).json" + test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json" result = put_file(storage, test_key, "url-test") @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/") @@ -576,8 +574,8 @@ using URIs "testbucket" ) - test_key = "testrunner-presign-$(randstring(8)).json" - test_data = "{\"presign\": true, \"test\": \"$(randstring(12))\"}" + test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json" + test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}" put_file(storage, test_key, test_data) url = s3_sign_url(storage, test_key, 300) diff --git a/test/winebottle.jpg b/test/winebottle.jpg new file mode 100644 index 0000000..7de5a42 Binary files /dev/null and b/test/winebottle.jpg differ