Compare commits
38 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ton
|
19d2a6dc9e | ||
|
ton
|
23b23cca9a | ||
|
ton
|
046d3ce4e6 | ||
|
ton
|
4f79b444d2 | ||
|
ton
|
599765d7ef | ||
|
ton
|
9ad732c436 | ||
|
ton
|
25ca9b96a2 | ||
|
ton
|
2827d22616 | ||
|
ton
|
151e4e74cf | ||
|
ton
|
42b66366ae | ||
|
ton
|
b0761fae75 | ||
|
ton
|
f9e952e5ed | ||
|
ton
|
397885511a | ||
|
ton
|
ee8ade1189 | ||
|
ton
|
20c51cf39f | ||
|
ton
|
f09a53b6c3 | ||
|
ton
|
c078948a7c | ||
|
ton
|
3780eb41df | ||
|
ton
|
c28b7ebf84 | ||
|
ton
|
27fd52fa71 | ||
|
ton
|
f88656172c | ||
|
ton
|
5c1b70107a | ||
|
ton
|
2b2b7a378e | ||
|
ton
|
3fb7457f61 | ||
|
ton
|
1fc0f790d9 | ||
|
ton
|
a559824faa | ||
|
ton
|
d74635c47a | ||
|
ton
|
9d6a943503 | ||
|
ton
|
c1c83eba77 | ||
|
ton
|
dee5649254 | ||
|
ton
|
fc62098071 | ||
|
ton
|
99996c1b04 | ||
|
ton
|
4466320927 | ||
|
ton
|
ff757c19dd | ||
|
ton
|
c4741140f5 | ||
|
ton
|
27703c3324 | ||
|
ton
|
86b8582584 | ||
|
ton
|
32cd21e1ee |
@@ -0,0 +1,41 @@
|
||||
# 1. Garage Web Admin UI
|
||||
garage-ui.yiem.cc {
|
||||
reverse_proxy 192.168.88.106:3909
|
||||
}
|
||||
|
||||
# 2. Garage S3 API (Path-Style & Single Domain)
|
||||
s3-api.yiem.cc {
|
||||
# CORS: handle OPTIONS preflight
|
||||
@options method OPTIONS {
|
||||
header Access-Control-Request-Method *
|
||||
}
|
||||
handle @options {
|
||||
respond * 204
|
||||
}
|
||||
# CORS headers on all responses
|
||||
header {
|
||||
Access-Control-Allow-Origin "*"
|
||||
Access-Control-Allow-Methods "GET, PUT, POST, DELETE, OPTIONS"
|
||||
Access-Control-Allow-Headers "*"
|
||||
Access-Control-Max-Age "86400"
|
||||
}
|
||||
reverse_proxy 192.168.88.106:3900 {
|
||||
header_up Host {host}
|
||||
}
|
||||
}
|
||||
|
||||
# 3. Garage S3 WEB (Path-Style & Single Domain)
|
||||
s3-web.yiem.cc {
|
||||
# /bucket -> /bucket/
|
||||
@bucket_root path_regexp bucket_root ^/([^/]+)$
|
||||
redir @bucket_root /{http.regexp.bucket_root.1}/ 301
|
||||
|
||||
# /bucket/... -> Garage
|
||||
@bucket path_regexp bucket ^/([^/]+)(/.*)?$
|
||||
|
||||
rewrite @bucket {http.regexp.bucket.2}
|
||||
|
||||
reverse_proxy 192.168.88.106:3902 {
|
||||
header_up Host {http.regexp.bucket.1}.s3-web.yiem.cc
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
|
||||
julia_version = "1.12.7"
|
||||
manifest_format = "2.0"
|
||||
project_hash = "4450296e6bb4e5e245de62f43b73b263b215aa5d"
|
||||
project_hash = "c54613132b678b04efd2a70a95fabd5b0cdd02a2"
|
||||
|
||||
[[deps.AWS]]
|
||||
deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MD5", "Mocking", "OrderedCollections", "Random", "SHA", "ScopedValues", "Sockets", "URIs", "UUIDs", "XMLDict"]
|
||||
@@ -306,7 +306,7 @@ version = "1.2.0"
|
||||
deps = ["AWS", "AWSS3", "CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "Graphs", "HTTP", "JSON", "LibPQ", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "StringDistances", "UUIDs"]
|
||||
path = "."
|
||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||
version = "0.6.0"
|
||||
version = "0.6.1"
|
||||
|
||||
[[deps.GitHub]]
|
||||
deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"]
|
||||
|
||||
+10
-2
@@ -1,11 +1,12 @@
|
||||
name = "GeneralUtils"
|
||||
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||
version = "0.6.1"
|
||||
authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
|
||||
version = "0.6.10"
|
||||
authors = ["tonaerospace <tonaerospace.etc@gmail.com"]
|
||||
|
||||
[deps]
|
||||
AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
|
||||
AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
|
||||
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
|
||||
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
|
||||
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
||||
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
||||
@@ -15,21 +16,28 @@ Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
|
||||
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
||||
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
|
||||
MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
|
||||
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
||||
OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
|
||||
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
||||
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
||||
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
|
||||
StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
|
||||
URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
|
||||
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||
|
||||
[compat]
|
||||
AWS = "1.99.0"
|
||||
AWSS3 = "0.11.6"
|
||||
Base64 = "1.11.0"
|
||||
Graphs = "1.14.0"
|
||||
HTTP = "2.5.0 - 2.9.9"
|
||||
JSON = "1.3.0 - 1.9.9"
|
||||
LibPQ = "1.18.0"
|
||||
MbedTLS = "1.1.10"
|
||||
NATS = "0.1.0"
|
||||
OrderedCollections = "2.0.1"
|
||||
Revise = "3.13.2"
|
||||
StringDistances = "1.0.0"
|
||||
URIs = "1.7.0"
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
# ==========================================
|
||||
# Garage Storage Engine Configuration
|
||||
# ==========================================
|
||||
|
||||
# Core storage directories
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
|
||||
# LMDB is recommended for production performance
|
||||
db_engine = "lmdb"
|
||||
metadata_auto_snapshot_interval = "6h"
|
||||
|
||||
# Multi-node ready setup:
|
||||
# Set to 1 for initial single node. Update to 3 when adding Node 2 and Node 3 later.
|
||||
replication_factor = 1
|
||||
compression_level = 2
|
||||
|
||||
# ==========================================
|
||||
# Network & Node Identity Configuration
|
||||
# ==========================================
|
||||
|
||||
# Bind to all interfaces for host network mode
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
|
||||
# IMPORTANT: Set to Node 1's publicly reachable IP or DNS name (e.g., 192.168.1.10:3901 or 203.0.113.10:3901)
|
||||
rpc_public_addr = "192.168.88.106:3901"
|
||||
|
||||
# Generate with: openssl rand -hex 32 (MUST BE IDENTICAL ON ALL NODES IN CLUSTER)
|
||||
rpc_secret = "3cf7a4d1da5d7b024a06db7fe6d73e2c160c6de640fda69b0b964d98678155a7"
|
||||
|
||||
# ==========================================
|
||||
# Service APIs
|
||||
# ==========================================
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "0.0.0.0:3900"
|
||||
root_domain = ".s3-api.yiem.cc"
|
||||
|
||||
[s3_web]
|
||||
bind_addr = "0.0.0.0:3902"
|
||||
root_domain = ".s3-web.yiem.cc"
|
||||
index = "index.html"
|
||||
|
||||
[k2v_api]
|
||||
api_bind_addr = "0.0.0.0:3904"
|
||||
|
||||
[admin]
|
||||
api_bind_addr = "0.0.0.0:3903"
|
||||
# Generate with: openssl rand -hex 16
|
||||
admin_token = "fd92f47a7c1ce8b072b06bb19dbc3fef"
|
||||
metrics_token = "27d29618cc436a23ec7d842392fdf74b"
|
||||
@@ -0,0 +1,48 @@
|
||||
services:
|
||||
garage:
|
||||
image: dxflrs/garage:v2.3.0
|
||||
container_name: garage
|
||||
restart: unless-stopped
|
||||
network_mode: "host"
|
||||
volumes:
|
||||
- ./garage.toml:/etc/garage.toml:ro
|
||||
- garage_meta:/var/lib/garage/meta
|
||||
- garage_data:/var/lib/garage/data
|
||||
environment:
|
||||
- RUST_LOG=garage=trace
|
||||
healthcheck:
|
||||
test: ["CMD", "/garage", "status"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
webui:
|
||||
image: khairul169/garage-webui:latest
|
||||
container_name: garage-webui
|
||||
restart: unless-stopped
|
||||
network_mode: "host"
|
||||
volumes:
|
||||
- ./garage.toml:/etc/garage.toml:ro
|
||||
environment:
|
||||
API_BASE_URL: "http://127.0.0.1:3903"
|
||||
S3_ENDPOINT_URL: "https://s3-api.yiem.cc"
|
||||
# Format: "username:bcrypt_hash" (generate via: htpasswd -nbBC 10 "admin" "yourpassword")
|
||||
AUTH_USER_PASS: "admin:$$2y$$10$$fhEucJHW8N5p8NL7Gv4Bi.FZVngbq/zatVDCjl.V.xNtj5q1PNUZW"
|
||||
depends_on:
|
||||
garage:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
garage_meta:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: ./data/meta
|
||||
garage_data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: ./data/storage
|
||||
@@ -0,0 +1,2 @@
|
||||
# dummy main.jl so i can do GeneralUtils precompile
|
||||
using GeneralUtils
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
This is my frontend code folder: /home/ton/docker-apps/sommpanion/agent-frontend/
|
||||
|
||||
Log from my frontend browser console:
|
||||
[NATS] Requesting from: sommpanion.5c1c85ad_44f8_49a7_a9a5_b7a7dface0db.agentBackend.v1.inbox payload: {"correlation_id":"5c1c85ad-44f8-49a7-a9a5-b7a7dface0db","msg_id":"FCA61ED2C63C4EDBB179378CEA010421","timestamp":"2026-09-05T11:44:52.208Z","send_to":"sommpanion.5c1c85ad_44f8_49a7_a9a5_b7a7dface0db.agentBackend.v1.inbox","msg_purpose":"chat","sender_name":"chat-webapp","sender_id":"5c1c85ad-44f8-49a7-a9a5-b7a7dface0db","receiver_name":"","receiver_id":"","reply_to":"sommpanion.frontend.agentfrontend.v1.inbox.5c1c85ad-44f8-49a7-a9a5-b7a7dface0db","reply_to_msg_id":"","broker_url":"wss://natsws.yiem.cc","metadata":{},"payloads":[{"id":"25F52E33F1284B58B4D76454C4DA2E96","dataname":"msg","payload_type":"dictionary","transport":"direct","encoding":"base64","size":97,"data":"eyJyb2xlIjoidXNlciIsImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6ImRvIHlvdSBoYXZlIHJlZCB3aW5lIGZyb20gRnJhbmNlIHVuZGVyIDEwMDA/In1dfQ==","metadata":{"payload_bytes":97}}]}
|
||||
natsClient.js:44 [NATS] Status event: pingTimer 1
|
||||
natsClient.js:174 [NATS] Request response: {broker_url: 'nats.yiem.cc', correlation_id: 'b9ebec8b-a4e3-432e-895d-8a099d92a5d1', metadata: {…}, msg_id: '860671db_fb1b_456f_a00f_c787c915e208', msg_purpose: 'chat', …}broker_url: "nats.yiem.cc"correlation_id: "b9ebec8b-a4e3-432e-895d-8a099d92a5d1"metadata: [[Prototype]]: Objectmsg_id: "860671db_fb1b_456f_a00f_c787c915e208"msg_purpose: "chat"payloads: Array(1)0: data: "https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf"dataname: "reply"encoding: "none"id: "13e9a0c4-fc4c-4d95-a9e0-6a9ae1f8219d"payload_type: "dictionary"size: 162786transport: "link"[[Prototype]]: Objectconstructor: ƒ Object()hasOwnProperty: ƒ hasOwnProperty()isPrototypeOf: ƒ isPrototypeOf()propertyIsEnumerable: ƒ propertyIsEnumerable()toLocaleString: ƒ toLocaleString()toString: ƒ toString()valueOf: ƒ valueOf()__defineGetter__: ƒ __defineGetter__()__defineSetter__: ƒ __defineSetter__()__lookupGetter__: ƒ __lookupGetter__()__lookupSetter__: ƒ __lookupSetter__()__proto__: (...)get __proto__: ƒ __proto__()set __proto__: ƒ __proto__()length: 1[[Prototype]]: Array(0)at: ƒ at()concat: ƒ concat()constructor: ƒ Array()copyWithin: ƒ copyWithin()entries: ƒ entries()every: ƒ every()fill: ƒ fill()filter: ƒ filter()find: ƒ find()findIndex: ƒ findIndex()findLast: ƒ findLast()findLastIndex: ƒ findLastIndex()flat: ƒ flat()flatMap: ƒ flatMap()forEach: ƒ forEach()includes: ƒ (item, from_index)indexOf: ƒ (item, from_index)join: ƒ join()keys: ƒ keys()lastIndexOf: ƒ (item, from_index)length: 0map: ƒ map()pop: ƒ pop()push: ƒ push()reduce: ƒ reduce()reduceRight: ƒ reduceRight()reverse: ƒ reverse()shift: ƒ shift()slice: ƒ slice()some: ƒ some()sort: ƒ sort()splice: ƒ splice()toLocaleString: ƒ toLocaleString()toReversed: ƒ toReversed()toSorted: ƒ toSorted()toSpliced: ƒ toSpliced()toString: ƒ toString()unshift: ƒ unshift()values: ƒ values()with: ƒ with()Symbol(Symbol.iterator): ƒ values()Symbol(Symbol.unscopables): {at: true, copyWithin: true, entries: true, fill: true, find: true, …}[[Prototype]]: Objectreceiver_id: ""receiver_name: ""reply_to: ""reply_to_msg_id: "FCA61ED2C63C4EDBB179378CEA010421"send_to: "_INBOX.P79XUSKPYH9E88Z9RM5QA1.P79XUSKPYH9E88Z9RM5QIN"sender_id: "828ef5e5-5d8e-47e2-8c48-736d99375af7"sender_name: "sommpanion_agent_backend"timestamp: "2026-09-05T18:47:37.858"[[Prototype]]: Object
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:36.857Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Processing received message
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:36.857Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] smartunpack: envelope has 1 payloads
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:36.857Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] smartunpack: Processing 1 payloads
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:36.858Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] smartunpack: Processing payload 1/1: dataname=reply, type=dictionary, transport=link
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:36.858Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Link transport - fetching 'reply' from URL: https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf
|
||||
:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
|
||||
msghandler-csr.js:274 GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
|
||||
fetchWithBackoff @ msghandler-csr.js:274
|
||||
smartunpack @ msghandler-csr.js:582
|
||||
processIncomingMessage @ Chat.svelte:136
|
||||
handleSend @ Chat.svelte:301
|
||||
await in handleSend
|
||||
handleSend @ ChatInput.svelte:21
|
||||
handleKeydown @ ChatInput.svelte:34
|
||||
handle_event_propagation @ events.js:278
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:36.962Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 1 failed: TypeError - Failed to fetch
|
||||
:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
|
||||
msghandler-csr.js:274 GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
|
||||
fetchWithBackoff @ msghandler-csr.js:274
|
||||
await in fetchWithBackoff
|
||||
smartunpack @ msghandler-csr.js:582
|
||||
processIncomingMessage @ Chat.svelte:136
|
||||
handleSend @ Chat.svelte:301
|
||||
await in handleSend
|
||||
handleSend @ ChatInput.svelte:21
|
||||
handleKeydown @ ChatInput.svelte:34
|
||||
handle_event_propagation @ events.js:278
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:37.110Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 2 failed: TypeError - Failed to fetch
|
||||
:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
|
||||
msghandler-csr.js:274 GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
|
||||
fetchWithBackoff @ msghandler-csr.js:274
|
||||
await in fetchWithBackoff
|
||||
smartunpack @ msghandler-csr.js:582
|
||||
processIncomingMessage @ Chat.svelte:136
|
||||
handleSend @ Chat.svelte:301
|
||||
await in handleSend
|
||||
handleSend @ ChatInput.svelte:21
|
||||
handleKeydown @ ChatInput.svelte:34
|
||||
handle_event_propagation @ events.js:278
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:37.359Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 3 failed: TypeError - Failed to fetch
|
||||
:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
|
||||
msghandler-csr.js:274 GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
|
||||
fetchWithBackoff @ msghandler-csr.js:274
|
||||
await in fetchWithBackoff
|
||||
smartunpack @ msghandler-csr.js:582
|
||||
processIncomingMessage @ Chat.svelte:136
|
||||
handleSend @ Chat.svelte:301
|
||||
await in handleSend
|
||||
handleSend @ ChatInput.svelte:21
|
||||
handleKeydown @ ChatInput.svelte:34
|
||||
handle_event_propagation @ events.js:278
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:37.801Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 4 failed: TypeError - Failed to fetch
|
||||
:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
|
||||
msghandler-csr.js:274 GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
|
||||
fetchWithBackoff @ msghandler-csr.js:274
|
||||
await in fetchWithBackoff
|
||||
smartunpack @ msghandler-csr.js:582
|
||||
processIncomingMessage @ Chat.svelte:136
|
||||
handleSend @ Chat.svelte:301
|
||||
await in handleSend
|
||||
handleSend @ ChatInput.svelte:21
|
||||
handleKeydown @ ChatInput.svelte:34
|
||||
handle_event_propagation @ events.js:278
|
||||
msghandler-csr.js:95 [2026-09-05T11:47:38.641Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 5 failed: TypeError - Failed to fetch
|
||||
Chat.svelte:162 Failed to process incoming message Error: Failed to fetch data after 5 attempts
|
||||
at fetchWithBackoff (msghandler-csr.js:293:11)
|
||||
at async Object.smartunpack (msghandler-csr.js:582:36)
|
||||
at async Module.track_reactivity_loss (async.js:179:14)
|
||||
at async processIncomingMessage (Chat.svelte:136:12)
|
||||
|
||||
Here is my backend log folder:
|
||||
/home/ton/docker-apps/sommpanion/agent-backend/log/
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+380
-69
@@ -1,42 +1,50 @@
|
||||
module garageS3
|
||||
|
||||
export
|
||||
GarageStorage,
|
||||
put_file,
|
||||
get_file,
|
||||
list_files,
|
||||
delete_file
|
||||
LifecycleExpiration, build_lifecycle_xml,
|
||||
GarageStorage, s3PutFile, s3GetFile, s3ListFiles, s3DeleteFile,
|
||||
s3SetLifecycleExpiration, s3GetLifecycle, s3DeleteLifecycle,
|
||||
s3SignUrl, s3SignUrlWeb
|
||||
|
||||
using AWS, AWSS3
|
||||
using AWS
|
||||
@service S3
|
||||
using AWSS3
|
||||
using Dates: Dates
|
||||
using URIs: URIs
|
||||
using OrderedCollections: OrderedDict
|
||||
using MbedTLS: digest, MD_SHA256
|
||||
using Base64: bytes2hex
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
|
||||
# this file use local AWS config
|
||||
|
||||
""" Example
|
||||
const storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
||||
"sommpanion-s3"
|
||||
storage1 = GeneralUtils.GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
||||
"sommpanion"
|
||||
)
|
||||
|
||||
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
||||
put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
|
||||
keys = list_files(storage)
|
||||
data = String(get_file(storage, "users-1004.json"))
|
||||
# use UUID as filename because GarageS3 requires unique filename for each uploaded object
|
||||
urls = GeneralUtils.s3PutFile(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
|
||||
key = GeneralUtils.s3ListFiles(storage)
|
||||
|
||||
println("Read back: ", data)
|
||||
println("Bucket keys: ", keys)
|
||||
println("Bucket keys: ", key)
|
||||
|
||||
# test with curl
|
||||
curl -v \
|
||||
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
||||
-H 'Host: testbucket.s3-web.yiem.cc' \
|
||||
http://192.168.88.106:3902/users-1002.json
|
||||
|
||||
# test with API
|
||||
data = String(GeneralUtils.s3GetFile(storage, key[1]))
|
||||
|
||||
# test with a browser
|
||||
https://s3-web.mydomain.com/my_bucket_name/users-1002.json
|
||||
# test with a browser (urls.web)
|
||||
https://s3-web.mydomain.com/my_bucket_name/users-1005.json
|
||||
|
||||
"""
|
||||
|
||||
@@ -44,13 +52,13 @@ https://s3-web.mydomain.com/my_bucket_name/users-1002.json
|
||||
# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
|
||||
# ===================================================================
|
||||
|
||||
struct SimpleCredentials
|
||||
mutable struct SimpleCredentials
|
||||
access_key_id::String
|
||||
secret_key::String
|
||||
token::String
|
||||
end
|
||||
|
||||
struct GarageConfig <: AWS.AbstractAWSConfig
|
||||
mutable struct GarageConfig <: AWS.AbstractAWSConfig
|
||||
endpoint::String
|
||||
region::String
|
||||
credentials::SimpleCredentials
|
||||
@@ -63,23 +71,6 @@ AWS.check_credentials(c::SimpleCredentials) = c
|
||||
AWS.region(aws::GarageConfig) = aws.region
|
||||
AWS.credentials(aws::GarageConfig) = aws.credentials
|
||||
|
||||
""" Generate the base URL for a Garage S3 service request.
|
||||
|
||||
# Arguments
|
||||
- `aws::GarageConfig`
|
||||
The Garage configuration containing the endpoint.
|
||||
- `service::String`
|
||||
The AWS service name (e.g., `"s3"`).
|
||||
- `region::String`
|
||||
The AWS region string.
|
||||
|
||||
# Return
|
||||
- `String`: The stripped endpoint URL (trailing slashes removed).
|
||||
"""
|
||||
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
||||
return strip(aws.endpoint, '/')
|
||||
end
|
||||
|
||||
""" Generate the full service URL including the resource path for a Garage S3 request.
|
||||
|
||||
# Arguments
|
||||
@@ -103,10 +94,6 @@ end
|
||||
# 2. Thread-Safe Storage Client Wrapper
|
||||
# ===================================================================
|
||||
|
||||
struct GarageStorage
|
||||
config::GarageConfig
|
||||
bucket::String
|
||||
end
|
||||
""" Create a thread-safe Garage S3 storage client.
|
||||
|
||||
# Arguments
|
||||
@@ -128,50 +115,95 @@ end
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb080154a2e5b19100b1b2c6e", "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", "sommpanion-s3")
|
||||
julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
||||
GarageStorage(GarageConfig(...), "sommpanion-s3")
|
||||
```
|
||||
"""
|
||||
mutable struct GarageStorage
|
||||
config::GarageConfig
|
||||
bucket::String
|
||||
end
|
||||
function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
|
||||
creds = SimpleCredentials(key, secret, "")
|
||||
config = GarageConfig(endpoint, region, creds)
|
||||
return GarageStorage(config, bucket)
|
||||
end
|
||||
|
||||
""" Upload an object to the Garage S3 bucket (callable struct).
|
||||
|
||||
# Arguments
|
||||
- `key::String`
|
||||
The object key (name) in the bucket.
|
||||
- `data::Union{String, Vector{UInt8}}`
|
||||
The data to upload.
|
||||
|
||||
# Return
|
||||
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
|
||||
|
||||
# Examples
|
||||
```jldoctest
|
||||
julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
|
||||
julia> s3PutFile1 = s3PutFile(storage1)
|
||||
julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}")
|
||||
Successfully uploaded: test.json
|
||||
julia> downloadUrl.api
|
||||
"https://s3-api.yiem.cc/sommpanion-s3/test.json"
|
||||
julia> downloadUrl.web
|
||||
"https://s3-web.yiem.cc/sommpanion-s3/test.json"
|
||||
```
|
||||
"""
|
||||
mutable struct s3PutFile
|
||||
storage::GarageStorage
|
||||
end
|
||||
function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}}
|
||||
)::NamedTuple{(:api, :web), Tuple{String, String}}
|
||||
AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
|
||||
println("Successfully uploaded: ", key)
|
||||
|
||||
# object url
|
||||
api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
|
||||
|
||||
# file download URL using browser
|
||||
web = replace(api, "s3-api" => "s3-web")
|
||||
|
||||
return (api=api, web=web)
|
||||
end
|
||||
|
||||
""" Upload an object to the Garage S3 bucket.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`
|
||||
The storage client instance.
|
||||
- `key::String`
|
||||
The object key (name) in the bucket. Must not contain slashes (`/`).
|
||||
The object key (name) in the bucket.
|
||||
- `data::Union{String, Vector{UInt8}}`
|
||||
The data to upload.
|
||||
|
||||
# Return
|
||||
- `Nothing` if the key contains slashes (upload aborted with warning).
|
||||
- Prints a success message if the upload completes.
|
||||
|
||||
# Notes
|
||||
- Slashes in keys are prohibited to prevent S3 listing issues. Use a flat naming
|
||||
convention (e.g., `"users-1002.json"` instead of `"users/1002.json"`).
|
||||
- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> put_file(storage, "test.json", "{\"status\": \"active\"}")
|
||||
julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}")
|
||||
Successfully uploaded: test.json
|
||||
julia> result.api
|
||||
"https://s3-api.yiem.cc/sommpanion-s3/test.json"
|
||||
julia> result.web
|
||||
"https://s3-web.yiem.cc/sommpanion-s3/test.json"
|
||||
```
|
||||
"""
|
||||
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})
|
||||
# Check if the key contains a slash (virtual folder character)
|
||||
if occursin('/', key)
|
||||
@warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
|
||||
return nothing
|
||||
end
|
||||
function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
|
||||
)::NamedTuple{(:api, :web), Tuple{String, String}}
|
||||
AWSS3.s3_put(storage.config, storage.bucket, key, data)
|
||||
println("Successfully uploaded: ", key)
|
||||
|
||||
# Proceed if the key is flat
|
||||
s3_put(storage.config, storage.bucket, key, data)
|
||||
println("Successfully uploaded: ", key)
|
||||
# object url
|
||||
api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
|
||||
|
||||
# file download URL using browser
|
||||
web = replace(api, "s3-api" => "s3-web")
|
||||
|
||||
return (api=api, web=web)
|
||||
end
|
||||
|
||||
""" Download an object from the Garage S3 bucket.
|
||||
@@ -187,12 +219,16 @@ end
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> data = String(get_file(storage, "test.json"))
|
||||
julia> data = String(s3GetFile(storage, "test.json"))
|
||||
"\"{\\\"status\\\": \\\"active\\\"}\""
|
||||
```
|
||||
"""
|
||||
function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
|
||||
return s3_get(storage.config, storage.bucket, key)
|
||||
function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
|
||||
try
|
||||
return AWSS3.s3_get(storage.config, storage.bucket, key; raw=true)
|
||||
catch
|
||||
return nothing
|
||||
end
|
||||
end
|
||||
|
||||
""" List all object keys in the Garage S3 bucket.
|
||||
@@ -206,16 +242,16 @@ end
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> keys = list_files(storage)
|
||||
julia> keys = s3ListFiles(storage)
|
||||
3-element Vector{String}:
|
||||
"test1.json"
|
||||
"test2.json"
|
||||
"users-1002.json"
|
||||
```
|
||||
"""
|
||||
function list_files(storage::GarageStorage)
|
||||
# Approach 2: s3_list_objects returns a Vector of Dicts with object details
|
||||
objects = s3_list_objects(storage.config, storage.bucket)
|
||||
function s3ListFiles(storage::GarageStorage)
|
||||
# Use delimiter="" to get all individual keys (no grouping by "directory")
|
||||
objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="")
|
||||
return [obj["Key"] for obj in objects]
|
||||
end
|
||||
|
||||
@@ -229,26 +265,301 @@ end
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
julia> delete_file(storage, "test.json")
|
||||
julia> s3DeleteFile(storage, "test.json")
|
||||
```
|
||||
"""
|
||||
function delete_file(storage::GarageStorage, key::String)
|
||||
s3_delete(storage.config, storage.bucket, key)
|
||||
function s3DeleteFile(storage::GarageStorage, key::String)
|
||||
AWSS3.s3_delete(storage.config, storage.bucket, key)
|
||||
end
|
||||
|
||||
"""
|
||||
s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String
|
||||
|
||||
Generate a pre-signed URL for an object in the Garage S3 bucket.
|
||||
|
||||
A pre-signed URL grants temporary access without AWS credentials. Signature is computed
|
||||
using AWS Signature Version 4 (HMAC-SHA256).
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`: Storage client from `GarageStorage(endpoint, key, secret, bucket)`.
|
||||
- `key::String`: Object key. Supports nested paths like `"folder/subfolder/file.txt"`.
|
||||
- `seconds::Int`: URL validity in seconds. Default: 3600.
|
||||
|
||||
# Keyword Arguments
|
||||
- `verb::String = "GET"`: HTTP method (`"GET"` downloads, `"PUT"` uploads).
|
||||
- `content_type::String = "application/octet-stream"`: Required `Content-Type` header.
|
||||
For PUT, set to the file's MIME type (`"image/png"`, `"application/json"`, etc.).
|
||||
- `signature_version::String = "v4"`: Always use `"v4"` (v2 is deprecated).
|
||||
|
||||
# Return
|
||||
`String`: The pre-signed URL.
|
||||
|
||||
# Examples
|
||||
|
||||
```julia
|
||||
using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile
|
||||
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"sommpanion"
|
||||
)
|
||||
|
||||
# Upload — PUT with curl
|
||||
url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg")
|
||||
# curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' <url>
|
||||
|
||||
# Download — GET with curl
|
||||
url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET")
|
||||
# curl <url> → returns file content
|
||||
|
||||
# Upload JSON
|
||||
url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json")
|
||||
# curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' <url>
|
||||
|
||||
# Delete
|
||||
url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE")
|
||||
# curl -X DELETE url
|
||||
|
||||
# Multiple uploads (chat app)
|
||||
urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300;
|
||||
verb="PUT", content_type="image/png")
|
||||
for _ in 1:10]
|
||||
|
||||
# Signature differs by verb and content_type
|
||||
url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET")
|
||||
url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain")
|
||||
```
|
||||
|
||||
# Notes
|
||||
- URLs are valid until `seconds` expires; they can be used **multiple times** (not one-time).
|
||||
- Server clocks must be within ~5 minutes of the Garage server (UTC time in signature).
|
||||
- PUT uploads require the client's `Content-Type` to match `content_type`.
|
||||
- For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`.
|
||||
"""
|
||||
function s3SignUrl(
|
||||
storage::GarageStorage,
|
||||
key::String,
|
||||
seconds::Int=3600;
|
||||
verb::String="GET",
|
||||
content_type::String="application/octet-stream",
|
||||
signature_version::String="v4",
|
||||
)::String
|
||||
config = storage.config
|
||||
creds = config.credentials
|
||||
bucket = storage.bucket
|
||||
region = config.region
|
||||
endpoint = strip(config.endpoint, '/')
|
||||
scheme = "AWS4"
|
||||
algorithm = "HMAC-SHA256"
|
||||
terminator = "aws4_request"
|
||||
service = "s3"
|
||||
|
||||
now_datetime = Dates.now(Dates.UTC)
|
||||
datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
|
||||
date_stamp = Dates.format(now_datetime, "YYYYmmdd")
|
||||
|
||||
scope = "$date_stamp/$region/$service/$terminator"
|
||||
|
||||
headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host)
|
||||
if verb == "PUT"
|
||||
headers["Content-Type"] = content_type
|
||||
end
|
||||
sort!(headers; by=name -> lowercase(name))
|
||||
canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
|
||||
|
||||
query = OrderedDict{String,String}(
|
||||
"X-Amz-Expires" => string(seconds),
|
||||
"X-Amz-Algorithm" => "$scheme-$algorithm",
|
||||
"X-Amz-Credential" => "$(creds.access_key_id)/$scope",
|
||||
"X-Amz-Date" => datetime_stamp,
|
||||
"X-Amz-SignedHeaders" => canonical_header_names,
|
||||
)
|
||||
if !isempty(creds.token)
|
||||
query["X-Amz-Security-Token"] = creds.token
|
||||
end
|
||||
|
||||
canonical_headers = join(
|
||||
map(
|
||||
header -> "$(lowercase(header.first)):$(lowercase(header.second))\n",
|
||||
collect(headers),
|
||||
),
|
||||
)
|
||||
|
||||
resource_path = "/$bucket/$key"
|
||||
|
||||
# Build canonical query string: sort by key=value pair lexicographically,
|
||||
# URL-encoding each key and value (matching Garage's uri_encode)
|
||||
sorted_q = sort(
|
||||
[URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)],
|
||||
by=lowercase,
|
||||
)
|
||||
canonical_query = join(sorted_q, "&")
|
||||
|
||||
canonical_request = string(
|
||||
"$verb\n",
|
||||
resource_path,
|
||||
"\n",
|
||||
canonical_query,
|
||||
"\n",
|
||||
canonical_headers,
|
||||
"\n",
|
||||
canonical_header_names,
|
||||
"\n",
|
||||
"UNSIGNED-PAYLOAD",
|
||||
)
|
||||
|
||||
string_to_sign = string(
|
||||
"$scheme-$algorithm\n",
|
||||
"$datetime_stamp\n",
|
||||
"$scope\n",
|
||||
bytes2hex(digest(MD_SHA256, canonical_request)),
|
||||
)
|
||||
|
||||
key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key))
|
||||
key_date = digest(MD_SHA256, date_stamp, key_secret_bytes)
|
||||
key_region = digest(MD_SHA256, region, key_date)
|
||||
key_service = digest(MD_SHA256, service, key_region)
|
||||
key_signing = digest(MD_SHA256, terminator, key_service)
|
||||
signature = digest(MD_SHA256, string_to_sign, key_signing)
|
||||
|
||||
query["X-Amz-Signature"] = bytes2hex(signature)
|
||||
|
||||
query_string = URIs.escapeuri(query)
|
||||
return "$(endpoint)/$bucket/$key?$query_string"
|
||||
end
|
||||
|
||||
""" Generate a pre-signed URL using the Garage S3 web endpoint (s3-web).
|
||||
|
||||
Useful for direct browser access — URLs look like
|
||||
`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...`
|
||||
|
||||
Arguments and keyword arguments are the same as `s3SignUrl`.
|
||||
"""
|
||||
function s3SignUrlWeb(
|
||||
storage::GarageStorage,
|
||||
key::String,
|
||||
seconds::Int=3600;
|
||||
verb::String="GET",
|
||||
content_type::String="application/octet-stream",
|
||||
signature_version::String="v4",
|
||||
)::String
|
||||
web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web")
|
||||
web_creds = SimpleCredentials(
|
||||
storage.config.credentials.access_key_id,
|
||||
storage.config.credentials.secret_key,
|
||||
storage.config.credentials.token,
|
||||
)
|
||||
web_config = GarageConfig(web_endpoint, storage.config.region, web_creds)
|
||||
web_storage = GarageStorage(web_config, storage.bucket)
|
||||
return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version)
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# 3. Bucket Lifecycle Configuration
|
||||
# ===================================================================
|
||||
|
||||
""" Lifecycle expiration configuration for a bucket.
|
||||
|
||||
# Fields
|
||||
- `prefix::String`: Object key prefix this rule applies to.
|
||||
- `days::Int`: Number of days after object creation to expire/delete.
|
||||
- `enabled::Bool`: Whether the rule is active (default: `true`).
|
||||
- `id::String`: Unique identifier for the rule (default: auto-generated).
|
||||
"""
|
||||
mutable struct LifecycleExpiration
|
||||
prefix::String
|
||||
days::Int
|
||||
enabled::Bool
|
||||
id::String
|
||||
end
|
||||
function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="")
|
||||
rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id
|
||||
return LifecycleExpiration(prefix, days, enabled, rule_id)
|
||||
end
|
||||
|
||||
""" Build the XML body for a PutBucketLifecycleConfiguration request.
|
||||
|
||||
Supports a single rule with expiration by prefix.
|
||||
"""
|
||||
function build_lifecycle_xml(rule::LifecycleExpiration)::String
|
||||
status = rule.enabled ? "Enabled" : "Disabled"
|
||||
return """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>$(rule.id)</ID>
|
||||
<Filter>
|
||||
<Prefix>$(rule.prefix)</Prefix>
|
||||
</Filter>
|
||||
<Status>$status</Status>
|
||||
<Expiration>
|
||||
<Days>$(rule.days)</Days>
|
||||
</Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"""
|
||||
end
|
||||
|
||||
""" Set bucket lifecycle expiration rule.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`: The storage client instance.
|
||||
- `rule::LifecycleExpiration`: The lifecycle expiration rule.
|
||||
|
||||
# Returns
|
||||
- `Dict{String, Any}`: Parsed response from the server.
|
||||
|
||||
# Example
|
||||
```julia
|
||||
rule = LifecycleExpiration("uploads/", 30)
|
||||
s3SetLifecycleExpiration(storage, rule)
|
||||
```
|
||||
"""
|
||||
function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration)
|
||||
xml_body = build_lifecycle_xml(rule)
|
||||
args = Dict{String,Any}("body" => xml_body)
|
||||
return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config)
|
||||
end
|
||||
|
||||
""" Get the bucket lifecycle configuration.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`: The storage client instance.
|
||||
|
||||
# Returns
|
||||
- `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured.
|
||||
|
||||
# Example
|
||||
```julia
|
||||
config = s3GetLifecycle(storage)
|
||||
```
|
||||
"""
|
||||
function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
|
||||
try
|
||||
return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config)
|
||||
catch e
|
||||
if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration"
|
||||
return nothing
|
||||
end
|
||||
rethrow(e)
|
||||
end
|
||||
end
|
||||
|
||||
""" Delete the bucket lifecycle configuration.
|
||||
|
||||
# Arguments
|
||||
- `storage::GarageStorage`: The storage client instance.
|
||||
|
||||
# Returns
|
||||
- `Dict{String, Any}`: Parsed response from the server.
|
||||
|
||||
# Example
|
||||
```julia
|
||||
s3DeleteLifecycle(storage)
|
||||
```
|
||||
"""
|
||||
function s3DeleteLifecycle(storage::GarageStorage)
|
||||
return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config)
|
||||
end
|
||||
|
||||
end # module GarageS3
|
||||
|
||||
+102
-2
@@ -4,12 +4,12 @@ export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, rep
|
||||
findMatchingDictKey, randstring, randstrings, timeout,
|
||||
dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString,
|
||||
dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
|
||||
dictToString_numbering, extract_triple_backtick_text,
|
||||
dictToString_numbering, extract_triple_backtick_text, eventSink,
|
||||
countGivenWords, remove_french_accents, removestring,
|
||||
extractTextBetweenCharacter, extractTextBetweenString,
|
||||
convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
|
||||
|
||||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames
|
||||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, NATS, Dates
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
@@ -1173,6 +1173,106 @@ function removestring(text::String, removelist::Vector{String})::String
|
||||
end
|
||||
|
||||
|
||||
"""
|
||||
Create an event sink for logging messages and/or publishing to NATS or printing to console. File logging defaults to false.
|
||||
|
||||
# Fields
|
||||
- `natsConn`: NATS connection, or `nothing` for file-only sink
|
||||
- `topic`: NATS topic to publish to, or `nothing` for file-only sink
|
||||
- `service_name`: Optional prefix added to log entries
|
||||
- `log_file`: Path to the log file
|
||||
- `max_log_entries`: Maximum number of entries to keep in the log file
|
||||
|
||||
# Callable interface
|
||||
```julia
|
||||
(es::eventSink)(msg::String; log=false, publish=true, console=true) -> Nothing
|
||||
```
|
||||
|
||||
Three independent outputs — each defaults to `true` except `log`:
|
||||
- `console` (default `true`) → println to stdout
|
||||
- `publish` (default `true`) → publish to NATS topic (no-op if no connection)
|
||||
- `log` (default `false`) → append to `log_file`
|
||||
|
||||
# Examples
|
||||
```julia
|
||||
using NATS, Dates
|
||||
|
||||
conn = NATS.connect("nats.yiem.cc")
|
||||
|
||||
# ——— With NATS (console + NATS by default) ———
|
||||
es = eventSink(natsConn=conn, topic="agent.events", service_name="agent1")
|
||||
es("starting up") # console + NATS
|
||||
es("logged"; log=true) # console + file + NATS
|
||||
es("quiet"; console=false) # NATS only
|
||||
es("file+nats"; log=true, console=false) # file + NATS
|
||||
es("console-only"; publish=false, log=false) # console only
|
||||
|
||||
# ——— File-only sink (no NATS connection) ———
|
||||
es2 = eventSink(log_file="./log/eventSink2.md", service_name="worker")
|
||||
es2("ready") # console only
|
||||
es2("file-logged"; log=true) # console + file
|
||||
es2("file-only"; log=true, console=false) # file only
|
||||
es2("silent"; console=false) # nothing (no NATS, log=false, console=false)
|
||||
|
||||
NATS.drain(conn)
|
||||
```
|
||||
"""
|
||||
struct eventSink
|
||||
natsConn::Union{NATS.Connection, Nothing}
|
||||
topic::Union{String, Nothing}
|
||||
service_name::String
|
||||
log_file::String
|
||||
max_log_entries::Int
|
||||
end
|
||||
eventSink(; natsConn=nothing, topic=nothing, service_name="", log_file="./log/eventSink.md", max_log_entries=100) =
|
||||
eventSink(natsConn, topic, service_name, log_file, max_log_entries)
|
||||
function (es::eventSink)(msg::String; log=false, publish=true, console=true)
|
||||
timestamp = Dates.format(now(), "yyyy-mm-ddTHH:MM:SS.sss")
|
||||
svc_prefix = isempty(es.service_name) ? "" : "[$(es.service_name)] "
|
||||
entry = "- [$timestamp] $svc_prefix$msg\n"
|
||||
if console
|
||||
println("[$timestamp] $svc_prefix$msg")
|
||||
end
|
||||
if log
|
||||
log_dir = dirname(es.log_file)
|
||||
mkpath(log_dir)
|
||||
open(es.log_file, "a") do f
|
||||
write(f, entry)
|
||||
end
|
||||
# Rotate after write — keep only last max_log_entries entries
|
||||
content = read(es.log_file, String)
|
||||
entries = split(content, r"- \[", keepempty=false)
|
||||
if length(entries) > es.max_log_entries
|
||||
rotate!(es)
|
||||
end
|
||||
end
|
||||
if publish && es.topic !== nothing && es.natsConn !== nothing
|
||||
try
|
||||
NATS.publish(es.natsConn, es.topic, entry)
|
||||
catch e
|
||||
@warn "eventSink NATS publish failed" exception=e
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
rotate!(es::eventSink) -> Nothing
|
||||
|
||||
Rewrite the log file keeping only the last `max_log_entries` entries.
|
||||
Call after writing when the entry count reaches the limit.
|
||||
"""
|
||||
function rotate!(es::eventSink)
|
||||
content = read(es.log_file, String)
|
||||
entries = split(content, r"- \[", keepempty=false)
|
||||
open(es.log_file, "w") do f
|
||||
write(f, "- [")
|
||||
for e in entries[2:end]
|
||||
write(f, "- [", e)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
curl -X PUT -H "Content-Type: application/json" \
|
||||
-d '{"test": "data"}' \
|
||||
"https://s3-api.yiem.cc/sommpanion/pcimage.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T103745Z&X-Amz-Expires=300&X-Amz-Security-Token=&X-Amz-SignedHeaders=host&X-Amz-Signature=0e195a48f824a2f8ffcb06461396cd6a8bddd5b61d93e9006f947ae10ecf525f"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
|
||||
-H 'Host: s3-api.yiem.cc' \
|
||||
-H 'Content-Type: image/jpeg' \
|
||||
"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T030315Z&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=1107a31f36f8b8ceb9663d69b9a7be7733df14d019e700e750944cbdad50df5d"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
read
|
||||
./garage.toml
|
||||
./Caddyfile
|
||||
./src/garageS3.jl
|
||||
|
||||
|
||||
|
||||
julia> url = GeneralUtils.s3_sign_url(storage1, "photo.jpg", 36000; verb="PUT", content_type="image/jpeg")
|
||||
"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T145004Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=92cd634dbb0f66db4b8b039ef5939c138203de260e8342fe27ef78979d90a824"
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
ton@server12vm-dev:~/docker-apps/sommpanion/etc/AWS.jl$
|
||||
curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
|
||||
-H 'Host: s3-api.yiem.cc' \
|
||||
-H 'Content-Type: image/jpeg' \
|
||||
"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02"
|
||||
|
||||
Note: Unnecessary use of -X or --request, PUT is already inferred.
|
||||
* Trying 110.77.160.154:443...
|
||||
* Connected to s3-api.yiem.cc (110.77.160.154) port 443 (#0)
|
||||
* ALPN: offers h2,http/1.1
|
||||
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
|
||||
* CAfile: /etc/ssl/certs/ca-certificates.crt
|
||||
* CApath: /etc/ssl/certs
|
||||
* TLSv1.3 (IN), TLS handshake, Server hello (2):
|
||||
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
|
||||
* TLSv1.3 (IN), TLS handshake, Certificate (11):
|
||||
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
|
||||
* TLSv1.3 (IN), TLS handshake, Finished (20):
|
||||
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
|
||||
* TLSv1.3 (OUT), TLS handshake, Finished (20):
|
||||
* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256
|
||||
* ALPN: server accepted h2
|
||||
* Server certificate:
|
||||
* subject: CN=s3-api.yiem.cc
|
||||
* start date: Aug 27 04:27:02 2026 GMT
|
||||
* expire date: Nov 25 04:27:01 2026 GMT
|
||||
* subjectAltName: host "s3-api.yiem.cc" matched cert's "s3-api.yiem.cc"
|
||||
* issuer: C=US; O=Let's Encrypt; CN=YE2
|
||||
* SSL certificate verify ok.
|
||||
* using HTTP/2
|
||||
* h2h3 [:method: PUT]
|
||||
* h2h3 [:path: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02]
|
||||
* h2h3 [:scheme: https]
|
||||
* h2h3 [:authority: s3-api.yiem.cc]
|
||||
* h2h3 [user-agent: curl/7.88.1]
|
||||
* h2h3 [accept: */*]
|
||||
* h2h3 [content-type: image/jpeg]
|
||||
* h2h3 [content-length: 439490]
|
||||
* Using Stream ID: 1 (easy handle 0x559a08824c80)
|
||||
> PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 HTTP/2
|
||||
> Host: s3-api.yiem.cc
|
||||
> user-agent: curl/7.88.1
|
||||
> accept: */*
|
||||
> content-type: image/jpeg
|
||||
> content-length: 439490
|
||||
>
|
||||
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
|
||||
* We are completely uploaded and fine
|
||||
< HTTP/2 403
|
||||
< access-control-allow-origin: *
|
||||
< alt-svc: h3=":443"; ma=2592000
|
||||
< content-type: application/xml
|
||||
< date: Fri, 04 Sep 2026 15:23:13 GMT
|
||||
< via: 1.1 Caddy
|
||||
< content-length: 190
|
||||
<
|
||||
* Connection #0 to host s3-api.yiem.cc left intact
|
||||
<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message>Forbidden: Invalid signature</Message><Resource>/sommpanion/photo.jpg</Resource><Region>garage</Region></Error>
|
||||
|
||||
2026-09-04T15:23:13.796037Z INFO garage_api_common::generic_server: 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02
|
||||
2026-09-04T15:23:13.796077Z DEBUG garage_api_common::generic_server: Request { method: PUT, uri: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02, version: HTTP/1.1, headers: {"host": "s3-api.yiem.cc", "user-agent": "curl/7.88.1", "content-length": "439490", "accept": "*/*", "content-type": "image/jpeg", "via": "2.0 Caddy", "x-forwarded-for": "110.77.160.154", "x-forwarded-host": "s3-api.yiem.cc", "x-forwarded-proto": "https", "accept-encoding": "gzip"}, body: Body(Streaming) }
|
||||
2026-09-04T15:23:13.796181Z DEBUG garage_api_common::generic_server: Endpoint: PutObject
|
||||
2026-09-04T15:23:13.796461Z INFO garage_api_common::generic_server: error 403 Forbidden, Forbidden: Invalid signature in response to 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+679
-25
@@ -1,39 +1,693 @@
|
||||
using Test
|
||||
using GeneralUtils: detect_keyword
|
||||
using GeneralUtils
|
||||
using GeneralUtils.garageS3
|
||||
using HTTP
|
||||
|
||||
@testset "detect_keyword tests" begin
|
||||
@test detect_keyword(["test"], "this is a test") == Dict("test" => 1)
|
||||
@testset "garageS3.jl" begin
|
||||
|
||||
@test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1)
|
||||
# --- SimpleCredentials ---
|
||||
|
||||
@test detect_keyword(["cat"], "category") == Dict("cat" => 1)
|
||||
@testset "SimpleCredentials construction" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken")
|
||||
@test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE"
|
||||
@test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
@test creds.token == "sessiontoken"
|
||||
end
|
||||
|
||||
@test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0)
|
||||
@testset "SimpleCredentials AWS extensions" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1")
|
||||
@test GeneralUtils.garageS3.AWS.credentials(creds) === creds
|
||||
@test GeneralUtils.garageS3.AWS.check_credentials(creds) === creds
|
||||
@test GeneralUtils.garageS3.AWS.refresh!(creds; force=false) === creds
|
||||
@test GeneralUtils.garageS3.AWS.refresh!(creds) === creds
|
||||
end
|
||||
|
||||
@test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0)
|
||||
# --- GarageConfig ---
|
||||
|
||||
@test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1)
|
||||
@testset "GarageConfig construction" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds)
|
||||
@test config.endpoint == "https://s3-api.example.com"
|
||||
@test config.region == "us-east-1"
|
||||
@test config.credentials === creds
|
||||
@test config isa GeneralUtils.garageS3.AWS.AbstractAWSConfig
|
||||
end
|
||||
|
||||
@test detect_keyword(["word"], "") == Dict("word" => 0)
|
||||
@testset "GarageConfig AWS extensions" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||
@test GeneralUtils.garageS3.AWS.region(config) == "garage"
|
||||
@test GeneralUtils.garageS3.AWS.credentials(config) === creds
|
||||
end
|
||||
|
||||
@test detect_keyword(String[], "some text") == Dict{String, Integer}()
|
||||
# --- GeneralUtils.garageS3.AWS.generate_service_url ---
|
||||
|
||||
@test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1)
|
||||
@testset "generate_service_url with resource starting with /" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||
url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/mybucket/key")
|
||||
@test url == "https://s3-api.example.com/mybucket/key"
|
||||
end
|
||||
|
||||
multiline_text = """
|
||||
first line
|
||||
second line
|
||||
first word
|
||||
"""
|
||||
@test detect_keyword(["first"], multiline_text) == Dict("first" => 2)
|
||||
@testset "generate_service_url with resource NOT starting with /" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||
url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "mybucket/key")
|
||||
@test url == "https://s3-api.example.com/mybucket/key"
|
||||
end
|
||||
|
||||
@test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1)
|
||||
@testset "generate_service_url strips trailing slashes from endpoint" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds)
|
||||
url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "https://s3-api.example.com/bucket/key"
|
||||
end
|
||||
|
||||
@testset "generate_service_url with multiple trailing slashes" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds)
|
||||
url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "https://s3-api.example.com/bucket/key"
|
||||
end
|
||||
|
||||
@testset "generate_service_url preserves http endpoint" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds)
|
||||
url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "http://localhost:3900/bucket/key"
|
||||
end
|
||||
|
||||
@testset "generate_service_url with http endpoint and trailing slash" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds)
|
||||
url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "http://localhost:3900/bucket/key"
|
||||
end
|
||||
|
||||
# --- GarageStorage ---
|
||||
|
||||
@testset "GarageStorage default region" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket")
|
||||
@test storage.bucket == "mybucket"
|
||||
@test storage.config.region == "garage"
|
||||
@test storage.config.endpoint == "https://s3-api.example.com"
|
||||
end
|
||||
|
||||
@testset "GarageStorage custom region" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region")
|
||||
@test storage.config.region == "custom-region"
|
||||
end
|
||||
|
||||
@testset "GarageStorage endpoint without https" begin
|
||||
storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket")
|
||||
@test storage.config.endpoint == "http://localhost:3900"
|
||||
end
|
||||
|
||||
@testset "GarageStorage credentials are SimpleCredentials" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket")
|
||||
@test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials
|
||||
@test storage.config.credentials.access_key_id == "mykey"
|
||||
@test storage.config.credentials.secret_key == "mysecret"
|
||||
@test storage.config.credentials.token == ""
|
||||
end
|
||||
|
||||
@testset "GarageStorage returns correct struct type" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
@test storage isa GarageStorage
|
||||
end
|
||||
|
||||
# --- s3PutFile callable struct ---
|
||||
|
||||
@testset "s3PutFile callable struct type is correct" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
uploader = s3PutFile(storage)
|
||||
@test uploader isa s3PutFile
|
||||
end
|
||||
|
||||
@testset "s3PutFile callable struct construction stores storage" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
uploader = s3PutFile(storage)
|
||||
@test uploader.storage === storage
|
||||
end
|
||||
|
||||
# --- Integration tests with real Garage server ---
|
||||
|
||||
@testset "integration: full CRUD lifecycle" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}"
|
||||
|
||||
# Test s3PutFile(storage, key, data) - successful upload
|
||||
result = s3PutFile(storage, test_key, test_data)
|
||||
@test result !== nothing
|
||||
@test haskey(result, :api)
|
||||
@test haskey(result, :web)
|
||||
@test occursin("testbucket", result.api)
|
||||
@test occursin("s3-api", result.api)
|
||||
@test occursin("s3-web", result.web)
|
||||
|
||||
# Test s3GetFile - retrieve the uploaded data
|
||||
downloaded = s3GetFile(storage, test_key)
|
||||
@test downloaded !== nothing
|
||||
@test downloaded isa Vector{UInt8}
|
||||
@test String(downloaded) == test_data
|
||||
|
||||
# Test s3ListFiles - verify key appears in listing
|
||||
keys = s3ListFiles(storage)
|
||||
@test test_key in keys
|
||||
|
||||
# Test s3PutFile callable struct
|
||||
test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json"
|
||||
callable_result = s3PutFile(storage)(test_key2, "callable-data")
|
||||
@test callable_result !== nothing
|
||||
@test callable_result.api != ""
|
||||
@test callable_result.web != ""
|
||||
|
||||
# Verify callable upload
|
||||
downloaded2 = s3GetFile(storage, test_key2)
|
||||
@test String(downloaded2) == "callable-data"
|
||||
|
||||
# Test s3DeleteFile
|
||||
s3DeleteFile(storage, test_key)
|
||||
deleted_data = s3GetFile(storage, test_key)
|
||||
@test deleted_data === nothing
|
||||
|
||||
# Cleanup callable test object
|
||||
s3DeleteFile(storage, test_key2)
|
||||
deleted_data2 = s3GetFile(storage, test_key2)
|
||||
@test deleted_data2 === nothing
|
||||
end
|
||||
|
||||
@testset "integration: nested key CRUD - single level" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"level\": 1, \"nested\": true}"
|
||||
|
||||
# Upload nested key
|
||||
result = s3PutFile(storage, test_key, test_data)
|
||||
@test result isa NamedTuple
|
||||
@test haskey(result, :api)
|
||||
@test haskey(result, :web)
|
||||
@test occursin("single/slash/", result.api)
|
||||
@test occursin("single/slash/", result.web)
|
||||
|
||||
# Download and verify
|
||||
downloaded = s3GetFile(storage, test_key)
|
||||
@test downloaded !== nothing
|
||||
@test String(downloaded) == test_data
|
||||
|
||||
# Verify in listing
|
||||
keys = s3ListFiles(storage)
|
||||
@test test_key in keys
|
||||
|
||||
# Delete and verify gone
|
||||
s3DeleteFile(storage, test_key)
|
||||
@test s3GetFile(storage, test_key) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: nested key CRUD - multi level" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}"
|
||||
|
||||
# Upload deeply nested key
|
||||
result = s3PutFile(storage, test_key, test_data)
|
||||
@test result isa NamedTuple
|
||||
@test occursin("deep/nested/folder/subfolder/", result.api)
|
||||
@test occursin("s3-api", result.api)
|
||||
@test occursin("s3-web", result.web)
|
||||
|
||||
# Download and verify
|
||||
downloaded = s3GetFile(storage, test_key)
|
||||
@test downloaded !== nothing
|
||||
@test String(downloaded) == test_data
|
||||
|
||||
# Verify in listing
|
||||
keys = s3ListFiles(storage)
|
||||
@test test_key in keys
|
||||
|
||||
# Delete and verify gone
|
||||
s3DeleteFile(storage, test_key)
|
||||
@test s3GetFile(storage, test_key) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: nested key with callable struct" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"callable\": true, \"nested\": true}"
|
||||
|
||||
uploader = s3PutFile(storage)
|
||||
result = uploader(test_key, test_data)
|
||||
@test result isa NamedTuple
|
||||
@test occursin("callable/nested/", result.api)
|
||||
|
||||
downloaded = s3GetFile(storage, test_key)
|
||||
@test String(downloaded) == test_data
|
||||
|
||||
s3DeleteFile(storage, test_key)
|
||||
@test s3GetFile(storage, test_key) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: s3GetFile returns nothing for missing key" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
result = s3GetFile(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "integration: binary data upload and download" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin"
|
||||
binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f]
|
||||
|
||||
result = s3PutFile(storage, test_key, binary_data)
|
||||
@test result !== nothing
|
||||
|
||||
downloaded = s3GetFile(storage, test_key)
|
||||
@test downloaded == binary_data
|
||||
|
||||
s3DeleteFile(storage, test_key)
|
||||
@test s3GetFile(storage, test_key) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: s3ListFiles returns vector of strings" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt"
|
||||
s3PutFile(storage, test_key, "list-test-data")
|
||||
|
||||
keys = s3ListFiles(storage)
|
||||
@test keys isa Vector{String}
|
||||
@test test_key in keys
|
||||
|
||||
s3DeleteFile(storage, test_key)
|
||||
end
|
||||
|
||||
@testset "integration: url format validation" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json"
|
||||
result = s3PutFile(storage, test_key, "url-test")
|
||||
|
||||
@test startswith(result.api, "https://s3-api.yiem.cc/testbucket/")
|
||||
@test occursin("/$test_key", result.api)
|
||||
@test startswith(result.web, "https://s3-web.yiem.cc/testbucket/")
|
||||
@test occursin("/$test_key", result.web)
|
||||
|
||||
s3DeleteFile(storage, test_key)
|
||||
end
|
||||
|
||||
# --- Lifecycle configuration tests ---
|
||||
|
||||
@testset "LifecycleExpiration construction with default ID" begin
|
||||
rule = LifecycleExpiration("uploads/", 30)
|
||||
@test rule.prefix == "uploads/"
|
||||
@test rule.days == 30
|
||||
@test rule.enabled == true
|
||||
@test rule.id == "expire-uploads_-30d"
|
||||
end
|
||||
|
||||
@testset "LifecycleExpiration construction with custom ID" begin
|
||||
rule = LifecycleExpiration("data/", 7; id="my-rule")
|
||||
@test rule.prefix == "data/"
|
||||
@test rule.days == 7
|
||||
@test rule.enabled == true
|
||||
@test rule.id == "my-rule"
|
||||
end
|
||||
|
||||
@testset "LifecycleExpiration with disabled flag" begin
|
||||
rule = LifecycleExpiration("temp/", 1; enabled=false)
|
||||
@test rule.enabled == false
|
||||
end
|
||||
|
||||
@testset "LifecycleExpiration with nested prefix" begin
|
||||
rule = LifecycleExpiration("deep/nested/", 90)
|
||||
@test rule.prefix == "deep/nested/"
|
||||
end
|
||||
|
||||
@testset "build_lifecycle_xml generates valid XML" begin
|
||||
rule = LifecycleExpiration("test/", 14; id="test-rule")
|
||||
xml = build_lifecycle_xml(rule)
|
||||
@test occursin("<?xml version=\"1.0\"", xml)
|
||||
@test occursin("<LifecycleConfiguration>", xml)
|
||||
@test occursin("<ID>test-rule</ID>", xml)
|
||||
@test occursin("<Prefix>test/</Prefix>", xml)
|
||||
@test occursin("<Status>Enabled</Status>", xml)
|
||||
@test occursin("<Days>14</Days>", xml)
|
||||
@test occursin("</LifecycleConfiguration>", xml)
|
||||
end
|
||||
|
||||
@testset "build_lifecycle_xml generates disabled rule XML" begin
|
||||
rule = LifecycleExpiration("temp/", 1; enabled=false)
|
||||
xml = build_lifecycle_xml(rule)
|
||||
@test occursin("<Status>Disabled</Status>", xml)
|
||||
end
|
||||
|
||||
@testset "integration: lifecycle CRUD - set and get" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
# Should return nothing when no lifecycle is set
|
||||
lc = s3GetLifecycle(storage)
|
||||
@test lc === nothing
|
||||
|
||||
# Set lifecycle
|
||||
rule = LifecycleExpiration("lifecycle-test/", 45)
|
||||
s3SetLifecycleExpiration(storage, rule)
|
||||
|
||||
# Should return configuration after setting
|
||||
lc = s3GetLifecycle(storage)
|
||||
@test lc !== nothing
|
||||
@test haskey(lc, "Rule")
|
||||
|
||||
# Delete lifecycle
|
||||
s3DeleteLifecycle(storage)
|
||||
|
||||
# Should return nothing after deletion
|
||||
lc = s3GetLifecycle(storage)
|
||||
@test lc === nothing
|
||||
end
|
||||
|
||||
@testset "integration: lifecycle with nested prefix" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule")
|
||||
s3SetLifecycleExpiration(storage, rule)
|
||||
|
||||
lc = s3GetLifecycle(storage)
|
||||
@test lc !== nothing
|
||||
@test haskey(lc, "Rule")
|
||||
|
||||
s3DeleteLifecycle(storage)
|
||||
@test s3GetLifecycle(storage) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: lifecycle enabled/disabled toggle" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
# Set enabled rule
|
||||
rule1 = LifecycleExpiration("toggle/", 10; enabled=true)
|
||||
s3SetLifecycleExpiration(storage, rule1)
|
||||
lc = s3GetLifecycle(storage)
|
||||
@test lc !== nothing
|
||||
|
||||
# Override with disabled rule
|
||||
rule2 = LifecycleExpiration("toggle/", 10; enabled=false)
|
||||
s3SetLifecycleExpiration(storage, rule2)
|
||||
lc = s3GetLifecycle(storage)
|
||||
@test lc !== nothing
|
||||
|
||||
s3DeleteLifecycle(storage)
|
||||
end
|
||||
|
||||
# --- Pre-signed URL tests ---
|
||||
|
||||
@testset "s3SignUrl returns a string" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url = s3SignUrl(storage, "test.json", 3600)
|
||||
@test url isa String
|
||||
@test !isempty(url)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl URL format contains correct endpoint, bucket, key" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url = s3SignUrl(storage, "mykey.json", 1800)
|
||||
@test occursin("https://s3-api.yiem.cc", url)
|
||||
@test occursin("/testbucket/mykey.json?", url)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl contains required query parameters" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url = s3SignUrl(storage, "test.json", 3600)
|
||||
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url)
|
||||
@test occursin("X-Amz-Expires=3600", url)
|
||||
@test occursin("X-Amz-Credential=", url)
|
||||
@test occursin("X-Amz-Date=", url)
|
||||
@test occursin("X-Amz-SignedHeaders=", url)
|
||||
@test occursin("X-Amz-Signature=", url)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl with custom expiry duration" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url = s3SignUrl(storage, "test.json", 600)
|
||||
@test occursin("X-Amz-Expires=600", url)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl with GET verb produces different signature than default" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url_get = s3SignUrl(storage, "test.json", 3600; verb="GET")
|
||||
url_default = s3SignUrl(storage, "test.json", 3600)
|
||||
@test occursin("X-Amz-Signature=", url_get)
|
||||
@test occursin("X-Amz-Signature=", url_default)
|
||||
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl with PUT verb and content_type" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url_put = s3SignUrl(storage, "upload.json", 300; verb="PUT", content_type="application/json")
|
||||
url_get = s3SignUrl(storage, "upload.json", 300; verb="GET")
|
||||
@test occursin("X-Amz-Signature=", url_put)
|
||||
@test occursin("X-Amz-Signature=", url_get)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl with special characters in key" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
test_key = "folder/sub folder/file with spaces.json"
|
||||
url = s3SignUrl(storage, test_key, 3600)
|
||||
@test occursin("testbucket/", url)
|
||||
end
|
||||
|
||||
@testset "s3SignUrl with deeply nested key" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
test_key = "a/b/c/d/e/deep-file.json"
|
||||
url = s3SignUrl(storage, test_key, 3600)
|
||||
@test occursin("/testbucket/a/b/c/d/e/deep-file.json", url)
|
||||
end
|
||||
|
||||
@testset "integration: pre-signed URL can download existing file" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}"
|
||||
s3PutFile(storage, test_key, test_data)
|
||||
|
||||
url = s3SignUrl(storage, test_key, 300)
|
||||
@test occursin("https://s3-api.yiem.cc/testbucket/", url)
|
||||
|
||||
s3DeleteFile(storage, test_key)
|
||||
end
|
||||
|
||||
# --- s3SignUrl end-to-end: PUT upload, GET download, DELETE ---
|
||||
|
||||
@testset "integration: s3SignUrl PUT upload via pre-signed URL" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-signurl-put-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"signed\": true, \"upload\": \"put\"}"
|
||||
content_type = "application/json"
|
||||
|
||||
# Generate pre-signed URL for PUT
|
||||
put_url = s3SignUrl(storage, test_key, 300; verb="PUT", content_type=content_type)
|
||||
@test occursin("X-Amz-Signature=", put_url)
|
||||
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", put_url)
|
||||
@test occursin("X-Amz-SignedHeaders=", put_url)
|
||||
|
||||
# Upload via HTTP request using the signed URL
|
||||
response = HTTP.request(
|
||||
"PUT",
|
||||
put_url,
|
||||
["Content-Type" => content_type],
|
||||
test_data
|
||||
)
|
||||
@test response.status == 200
|
||||
|
||||
# Verify upload via s3GetFile
|
||||
downloaded = s3GetFile(storage, test_key)
|
||||
@test downloaded !== nothing
|
||||
@test String(downloaded) == test_data
|
||||
|
||||
# Cleanup
|
||||
s3DeleteFile(storage, test_key)
|
||||
end
|
||||
|
||||
@testset "integration: s3SignUrl GET download via pre-signed URL" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-signurl-get-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"signed\": true, \"download\": \"get\"}"
|
||||
|
||||
# Upload via s3PutFile first
|
||||
s3PutFile(storage, test_key, test_data)
|
||||
|
||||
# Generate pre-signed URL for GET
|
||||
get_url = s3SignUrl(storage, test_key, 300; verb="GET")
|
||||
@test occursin("X-Amz-Signature=", get_url)
|
||||
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", get_url)
|
||||
|
||||
# Download via HTTP GET request using the signed URL
|
||||
response = HTTP.request("GET", get_url)
|
||||
@test response.status == 200
|
||||
@test String(response.body) == test_data
|
||||
|
||||
# Cleanup
|
||||
s3DeleteFile(storage, test_key)
|
||||
end
|
||||
|
||||
@testset "integration: s3SignUrl DELETE via pre-signed URL" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-signurl-delete-$(GeneralUtils.uuid4snakecase()).json"
|
||||
test_data = "{\"signed\": true, \"delete\": true}"
|
||||
|
||||
# Upload first
|
||||
s3PutFile(storage, test_key, test_data)
|
||||
|
||||
# Verify it exists
|
||||
@test s3GetFile(storage, test_key) !== nothing
|
||||
|
||||
# Generate pre-signed URL for DELETE
|
||||
delete_url = s3SignUrl(storage, test_key, 300; verb="DELETE")
|
||||
@test occursin("X-Amz-Signature=", delete_url)
|
||||
@test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", delete_url)
|
||||
|
||||
# Delete via HTTP DELETE request using the signed URL
|
||||
response = HTTP.request("DELETE", delete_url)
|
||||
@test response.status == 204
|
||||
|
||||
# Verify deletion
|
||||
@test s3GetFile(storage, test_key) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: s3SignUrlWeb generates web endpoint URL" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
url = s3SignUrlWeb(storage, "web-test.json", 3600)
|
||||
@test url isa String
|
||||
@test occursin("https://s3-web.yiem.cc", url)
|
||||
@test occursin("/testbucket/web-test.json?", url)
|
||||
@test occursin("X-Amz-Signature=", url)
|
||||
end
|
||||
|
||||
@test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1)
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Reference in New Issue
Block a user