update

Reviewed-on: #15

Caddyfile

@@ -0,0 +1,41 @@
+# 1. Garage Web Admin UI
+garage-ui.yiem.cc {
+    reverse_proxy 192.168.88.106:3909
+}
+
+# 2. Garage S3 API (Path-Style & Single Domain)
+s3-api.yiem.cc {
+    # CORS: handle OPTIONS preflight
+    @options method OPTIONS {
+        header Access-Control-Request-Method *
+    }
+    handle @options {
+        respond * 204
+    }
+    # CORS headers on all responses
+    header {
+        Access-Control-Allow-Origin "*"
+        Access-Control-Allow-Methods "GET, PUT, POST, DELETE, OPTIONS"
+        Access-Control-Allow-Headers "*"
+        Access-Control-Max-Age "86400"
+    }
+    reverse_proxy 192.168.88.106:3900 {
+        header_up Host {host}
+    }
+}
+
+# 3. Garage S3 WEB (Path-Style & Single Domain)
+s3-web.yiem.cc {
+    # /bucket -> /bucket/
+    @bucket_root path_regexp bucket_root ^/([^/]+)$
+    redir @bucket_root /{http.regexp.bucket_root.1}/ 301
+
+    # /bucket/... -> Garage
+    @bucket path_regexp bucket ^/([^/]+)(/.*)?$
+
+    rewrite @bucket {http.regexp.bucket.2}
+
+    reverse_proxy 192.168.88.106:3902 {
+        header_up Host {http.regexp.bucket.1}.s3-web.yiem.cc
+    }
+}

Project.toml

@@ -1,17 +1,43 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
-version = "0.2.3"
+version = "0.6.10"
+authors = ["tonaerospace <tonaerospace.etc@gmail.com"]
 
 [deps]
+AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
+AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
-MQTTClient = "985f35cc-2c3d-4943-b8c1-f0931d5f0959"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
+MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
+NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
+URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[compat]
+AWS = "1.99.0"
+AWSS3 = "0.11.6"
+Base64 = "1.11.0"
+Graphs = "1.14.0"
+HTTP = "2.5.0 - 2.9.9"
+JSON = "1.3.0 - 1.9.9"
+LibPQ = "1.18.0"
+MbedTLS = "1.1.10"
+NATS = "0.1.0"
+OrderedCollections = "2.0.1"
+Revise = "3.13.2"
+StringDistances = "1.0.0"
+URIs = "1.7.0"

README.md

@@ -0,0 +1,5 @@
+Todo:
+  - [WORKING] update with JSON
+
+  Change from previous version:
+  - replace JSON3 with JSON

codesnippet/nats.jl

@@ -0,0 +1,79 @@
+
+using NATS, JSON3
+connection  = NATS.connect("nats.yiem.cc:4222")
+sub1 = NATS.reply(connection, "some_subject"; queue_group="group1") do msg
+                  payload = copy(JSON3.read(msg.payload))
+                  println(payload)
+                  println(msg.reply_to)
+                  # publish(connection, msg.reply_to, "ACK")
+                  return JSON3.write(Dict(:a=>"wassup"))
+              end
+
+
+
+
+
+
+
+
+
+
+using NATS, JSON3, GeneralUtils
+connection  = NATS.connect("nats.yiem.cc:4222")
+
+msgMeta = GeneralUtils.generate_msgMeta(
+  "text2textinstruct_medium.inference.api.v1";
+  msgPurpose= "inference",
+  senderName= "yiemagent",
+  senderId= GeneralUtils.uuid4snakecase(),
+  receiverName= "text2textinstruct",
+)
+
+llmHttpTimeout = 60
+outgoingMsg = Dict(
+    :msgMeta=> msgMeta,
+    :payload=> Dict(
+      :text=> "Wassup buddy!",
+      :kwargs=> Dict(
+        :max_tokens=> 2048,
+        :stop=> ["<|im_end|>"],
+        :temperature=> 0.2,
+      ),
+      :llmHttpTimeout=>llmHttpTimeout,
+    )
+  )
+
+r = NATS.request(String, connection, "text2textinstruct_medium.inference.api.v1", 
+                JSON3.write(outgoingMsg); timer=Timer(llmHttpTimeout))
+
+
+
+
+
+
+
+
+
+
+
+using NATS, JSON3, GeneralUtils
+connection  = NATS.connect("nats.yiem.cc:4222")
+
+msgMeta = GeneralUtils.generate_msgMeta(
+  "tonpc.containerServices",
+  msgPurpose="reset container",
+  senderName= "",
+)
+
+outgoingMsg = Dict(
+  :msgMeta=> msgMeta,
+  :payload=> "docker container restart ollama-instance-2",
+)
+
+# may be I can't use NATS request inside NATS reply??
+r = NATS.request(String, connection, msgMeta[:sendTopic], JSON3.write(outgoingMsg); timer=Timer(10))
+
+
+
+
+

garage.toml

@@ -0,0 +1,52 @@
+# ==========================================
+# Garage Storage Engine Configuration
+# ==========================================
+
+# Core storage directories
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+
+# LMDB is recommended for production performance
+db_engine = "lmdb"
+metadata_auto_snapshot_interval = "6h"
+
+# Multi-node ready setup:
+# Set to 1 for initial single node. Update to 3 when adding Node 2 and Node 3 later.
+replication_factor = 1
+compression_level = 2
+
+# ==========================================
+# Network & Node Identity Configuration
+# ==========================================
+
+# Bind to all interfaces for host network mode
+rpc_bind_addr = "[::]:3901"
+
+# IMPORTANT: Set to Node 1's publicly reachable IP or DNS name (e.g., 192.168.1.10:3901 or 203.0.113.10:3901)
+rpc_public_addr = "192.168.88.106:3901"
+
+# Generate with: openssl rand -hex 32 (MUST BE IDENTICAL ON ALL NODES IN CLUSTER)
+rpc_secret = "3cf7a4d1da5d7b024a06db7fe6d73e2c160c6de640fda69b0b964d98678155a7"
+
+# ==========================================
+# Service APIs
+# ==========================================
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "0.0.0.0:3900"
+root_domain = ".s3-api.yiem.cc"
+
+[s3_web]
+bind_addr = "0.0.0.0:3902"
+root_domain = ".s3-web.yiem.cc"
+index = "index.html"
+
+[k2v_api]
+api_bind_addr = "0.0.0.0:3904"
+
+[admin]
+api_bind_addr = "0.0.0.0:3903"
+# Generate with: openssl rand -hex 16
+admin_token = "fd92f47a7c1ce8b072b06bb19dbc3fef"
+metrics_token = "27d29618cc436a23ec7d842392fdf74b"

garage_docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  garage:
+    image: dxflrs/garage:v2.3.0
+    container_name: garage
+    restart: unless-stopped
+    network_mode: "host"
+    volumes:
+      - ./garage.toml:/etc/garage.toml:ro
+      - garage_meta:/var/lib/garage/meta
+      - garage_data:/var/lib/garage/data
+    environment:
+      - RUST_LOG=garage=trace
+    healthcheck:
+      test: ["CMD", "/garage", "status"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+  webui:
+    image: khairul169/garage-webui:latest
+    container_name: garage-webui
+    restart: unless-stopped
+    network_mode: "host"
+    volumes:
+      - ./garage.toml:/etc/garage.toml:ro
+    environment:
+      API_BASE_URL: "http://127.0.0.1:3903"
+      S3_ENDPOINT_URL: "https://s3-api.yiem.cc"
+      # Format: "username:bcrypt_hash" (generate via: htpasswd -nbBC 10 "admin" "yourpassword")
+      AUTH_USER_PASS: "admin:$$2y$$10$$fhEucJHW8N5p8NL7Gv4Bi.FZVngbq/zatVDCjl.V.xNtj5q1PNUZW"
+    depends_on:
+      garage:
+        condition: service_healthy
+
+volumes:
+  garage_meta:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ./data/meta
+  garage_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ./data/storage

main.jl

@@ -0,0 +1,2 @@
+# dummy main.jl so i can do GeneralUtils precompile
+using GeneralUtils

my_AI_prompt.md

@@ -0,0 +1,147 @@
+This is my frontend code folder: /home/ton/docker-apps/sommpanion/agent-frontend/ 
+
+Log from my frontend browser console:
+[NATS] Requesting from: sommpanion.5c1c85ad_44f8_49a7_a9a5_b7a7dface0db.agentBackend.v1.inbox payload: {"correlation_id":"5c1c85ad-44f8-49a7-a9a5-b7a7dface0db","msg_id":"FCA61ED2C63C4EDBB179378CEA010421","timestamp":"2026-09-05T11:44:52.208Z","send_to":"sommpanion.5c1c85ad_44f8_49a7_a9a5_b7a7dface0db.agentBackend.v1.inbox","msg_purpose":"chat","sender_name":"chat-webapp","sender_id":"5c1c85ad-44f8-49a7-a9a5-b7a7dface0db","receiver_name":"","receiver_id":"","reply_to":"sommpanion.frontend.agentfrontend.v1.inbox.5c1c85ad-44f8-49a7-a9a5-b7a7dface0db","reply_to_msg_id":"","broker_url":"wss://natsws.yiem.cc","metadata":{},"payloads":[{"id":"25F52E33F1284B58B4D76454C4DA2E96","dataname":"msg","payload_type":"dictionary","transport":"direct","encoding":"base64","size":97,"data":"eyJyb2xlIjoidXNlciIsImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6ImRvIHlvdSBoYXZlIHJlZCB3aW5lIGZyb20gRnJhbmNlIHVuZGVyIDEwMDA/In1dfQ==","metadata":{"payload_bytes":97}}]}
+natsClient.js:44 [NATS] Status event: pingTimer 1
+natsClient.js:174 [NATS] Request response: {broker_url: 'nats.yiem.cc', correlation_id: 'b9ebec8b-a4e3-432e-895d-8a099d92a5d1', metadata: {…}, msg_id: '860671db_fb1b_456f_a00f_c787c915e208', msg_purpose: 'chat', …}broker_url: "nats.yiem.cc"correlation_id: "b9ebec8b-a4e3-432e-895d-8a099d92a5d1"metadata: [[Prototype]]: Objectmsg_id: "860671db_fb1b_456f_a00f_c787c915e208"msg_purpose: "chat"payloads: Array(1)0: data: "https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf"dataname: "reply"encoding: "none"id: "13e9a0c4-fc4c-4d95-a9e0-6a9ae1f8219d"payload_type: "dictionary"size: 162786transport: "link"[[Prototype]]: Objectconstructor: ƒ Object()hasOwnProperty: ƒ hasOwnProperty()isPrototypeOf: ƒ isPrototypeOf()propertyIsEnumerable: ƒ propertyIsEnumerable()toLocaleString: ƒ toLocaleString()toString: ƒ toString()valueOf: ƒ valueOf()__defineGetter__: ƒ __defineGetter__()__defineSetter__: ƒ __defineSetter__()__lookupGetter__: ƒ __lookupGetter__()__lookupSetter__: ƒ __lookupSetter__()__proto__: (...)get __proto__: ƒ __proto__()set __proto__: ƒ __proto__()length: 1[[Prototype]]: Array(0)at: ƒ at()concat: ƒ concat()constructor: ƒ Array()copyWithin: ƒ copyWithin()entries: ƒ entries()every: ƒ every()fill: ƒ fill()filter: ƒ filter()find: ƒ find()findIndex: ƒ findIndex()findLast: ƒ findLast()findLastIndex: ƒ findLastIndex()flat: ƒ flat()flatMap: ƒ flatMap()forEach: ƒ forEach()includes: ƒ (item, from_index)indexOf: ƒ (item, from_index)join: ƒ join()keys: ƒ keys()lastIndexOf: ƒ (item, from_index)length: 0map: ƒ map()pop: ƒ pop()push: ƒ push()reduce: ƒ reduce()reduceRight: ƒ reduceRight()reverse: ƒ reverse()shift: ƒ shift()slice: ƒ slice()some: ƒ some()sort: ƒ sort()splice: ƒ splice()toLocaleString: ƒ toLocaleString()toReversed: ƒ toReversed()toSorted: ƒ toSorted()toSpliced: ƒ toSpliced()toString: ƒ toString()unshift: ƒ unshift()values: ƒ values()with: ƒ with()Symbol(Symbol.iterator): ƒ values()Symbol(Symbol.unscopables): {at: true, copyWithin: true, entries: true, fill: true, find: true, …}[[Prototype]]: Objectreceiver_id: ""receiver_name: ""reply_to: ""reply_to_msg_id: "FCA61ED2C63C4EDBB179378CEA010421"send_to: "_INBOX.P79XUSKPYH9E88Z9RM5QA1.P79XUSKPYH9E88Z9RM5QIN"sender_id: "828ef5e5-5d8e-47e2-8c48-736d99375af7"sender_name: "sommpanion_agent_backend"timestamp: "2026-09-05T18:47:37.858"[[Prototype]]: Object
+msghandler-csr.js:95 [2026-09-05T11:47:36.857Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Processing received message
+msghandler-csr.js:95 [2026-09-05T11:47:36.857Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] smartunpack: envelope has 1 payloads
+msghandler-csr.js:95 [2026-09-05T11:47:36.857Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] smartunpack: Processing 1 payloads
+msghandler-csr.js:95 [2026-09-05T11:47:36.858Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] smartunpack: Processing payload 1/1: dataname=reply, type=dictionary, transport=link
+msghandler-csr.js:95 [2026-09-05T11:47:36.858Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Link transport - fetching 'reply' from URL: https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf
+:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
+msghandler-csr.js:274  GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
+fetchWithBackoff @ msghandler-csr.js:274
+smartunpack @ msghandler-csr.js:582
+processIncomingMessage @ Chat.svelte:136
+handleSend @ Chat.svelte:301
+await in handleSend
+handleSend @ ChatInput.svelte:21
+handleKeydown @ ChatInput.svelte:34
+handle_event_propagation @ events.js:278
+msghandler-csr.js:95 [2026-09-05T11:47:36.962Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 1 failed: TypeError - Failed to fetch
+:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
+msghandler-csr.js:274  GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
+fetchWithBackoff @ msghandler-csr.js:274
+await in fetchWithBackoff
+smartunpack @ msghandler-csr.js:582
+processIncomingMessage @ Chat.svelte:136
+handleSend @ Chat.svelte:301
+await in handleSend
+handleSend @ ChatInput.svelte:21
+handleKeydown @ ChatInput.svelte:34
+handle_event_propagation @ events.js:278
+msghandler-csr.js:95 [2026-09-05T11:47:37.110Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 2 failed: TypeError - Failed to fetch
+:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
+msghandler-csr.js:274  GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
+fetchWithBackoff @ msghandler-csr.js:274
+await in fetchWithBackoff
+smartunpack @ msghandler-csr.js:582
+processIncomingMessage @ Chat.svelte:136
+handleSend @ Chat.svelte:301
+await in handleSend
+handleSend @ ChatInput.svelte:21
+handleKeydown @ ChatInput.svelte:34
+handle_event_propagation @ events.js:278
+msghandler-csr.js:95 [2026-09-05T11:47:37.359Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 3 failed: TypeError - Failed to fetch
+:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
+msghandler-csr.js:274  GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
+fetchWithBackoff @ msghandler-csr.js:274
+await in fetchWithBackoff
+smartunpack @ msghandler-csr.js:582
+processIncomingMessage @ Chat.svelte:136
+handleSend @ Chat.svelte:301
+await in handleSend
+handleSend @ ChatInput.svelte:21
+handleKeydown @ ChatInput.svelte:34
+handle_event_propagation @ events.js:278
+msghandler-csr.js:95 [2026-09-05T11:47:37.801Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 4 failed: TypeError - Failed to fetch
+:5173/#chat:1 Access to fetch at 'https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf' from origin 'http://localhost:5173' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
+msghandler-csr.js:274  GET https://s3-api.yiem.cc/sommpanion/reply?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T114737Z&X-Amz-SignedHeaders=host&X-Amz-Signature=d5249d804b46114baf6e6436e2f8d8a22bc362422373a601de03ce8606edbbbf net::ERR_FAILED 200 (OK)
+fetchWithBackoff @ msghandler-csr.js:274
+await in fetchWithBackoff
+smartunpack @ msghandler-csr.js:582
+processIncomingMessage @ Chat.svelte:136
+handleSend @ Chat.svelte:301
+await in handleSend
+handleSend @ ChatInput.svelte:21
+handleKeydown @ ChatInput.svelte:34
+handle_event_propagation @ events.js:278
+msghandler-csr.js:95 [2026-09-05T11:47:38.641Z] [Correlation: b9ebec8b-a4e3-432e-895d-8a099d92a5d1] Attempt 5 failed: TypeError - Failed to fetch
+Chat.svelte:162 Failed to process incoming message Error: Failed to fetch data after 5 attempts
+    at fetchWithBackoff (msghandler-csr.js:293:11)
+    at async Object.smartunpack (msghandler-csr.js:582:36)
+    at async Module.track_reactivity_loss (async.js:179:14)
+    at async processIncomingMessage (Chat.svelte:136:12)
+
+Here is my backend log folder: 
+/home/ton/docker-apps/sommpanion/agent-backend/log/
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/GeneralUtils.jl

@@ -2,7 +2,7 @@ module GeneralUtils
     
 
 export  # struct
-        mqttClientInstance,
+        # mqttClientInstance,
         # function
         noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue 
 
@@ -21,19 +21,12 @@ using .llmUtil
 include("interface.jl")
 using .interface
 
+include("garageS3.jl")
+using .garageS3
 
 #------------------------------------------------------------------------------------------------100
 
-""" version 0.0.4
-    Todo:
-        - [*1] cartesianAssign for different matrix dimension
 
-    Change from version: 0.0.3
-        - 
-    
-    All features
-        
-"""
 
 
 

src/dbUtil.jl

@@ -1,14 +1,140 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
-using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames,
-      SHA
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
+      SHA, NATS, LibPQ
 using ..util
-
+#[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
+""" Execute SQL against a PostgreSQL database using LibPQ connection string.
 
+# Arguments
+- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
+- `sql::AbstractString`: SQL query to execute
+
+# Returns
+- `LibPQ.Result` on success, `nothing` on failure
+
+# Example
+```julia
+pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
+sql = "SELECT * FROM wine;"
+result = execute_postgres_sql(pg_conn_str, sql)
+```
+"""
+function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
+  db_connection = LibPQ.Connection(pg_conn_str)
+  result = nothing
+  try
+    result = LibPQ.execute(db_connection, sql)  
+  catch e
+    @error e
+    LibPQ.close(db_connection)
+  end
+  
+  LibPQ.close(db_connection)
+  return result
+end
+
+
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
+
+# Arguments
+- `text::AbstractString`: The input text to find similar records for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
+
+# Returns
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
+
+# Example
+```julia
+# Assume you have embedding and SQL execution functions
+text = "a rich structured red wine"
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
+```
+"""
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
+  executesql::Function, get_embedding; 
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
+end
+
+
+"""
+dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
+
+This function takes a dictionary and converts it into a PostgreSQL-compatible key-value string
+format suitable for storage in a TEXT field. The output format uses curly braces with comma-separated
+key-value pairs, where string values are quoted.
+
+# Function Workflow:
+1. Iterates through dictionary key-value pairs
+2. Handles nested dictionaries by recursively converting them
+3. Wraps string values in double quotes
+4. Formats numeric and other values without quotes
+5. Returns a PostgreSQL-compatible key-value string enclosed in curly braces
+
+# Arguments:
+  - `dict::Dict` - Dictionary containing key-value pairs to convert
+
+# Return:
+  - A String in PostgreSQL key-value format: "{key1: value1, key2: \"value2\", ...}"
+
+# Example
+```jldoctest
+julia> data = Dict{String, Any}(
+           "name" => "John",
+           "age" => 30,
+           "city" => "New York"
+       );
+
+julia> dictToPostgresKeyValueString(data)
+"{\"name\": \"John\", \"age\": 30, \"city\": \"New York\"}"
+```
+"""
 function dictToPostgresKeyValueString(dict)
   parts = []
   for (k, v) in dict
@@ -26,61 +152,71 @@ end
 
 
 
-""" Get characters between specified characters.
+""" generateInsertSQL - Generate SQL INSERT statement from dictionary data
 
-# Arguments
-  - `text::T`
-    a text being searched
-  - `startChar::Char`
-    start character
-  - `endChar::Char`
-    end character
-# Keyword Arguments
-  - `endCharLocation::String`
-    end character position after startChar. Can be "next" or "end". "next" means the closed 
-    endChar just after startChar. "end" means the furthest endChar. 
-  - `includeChar::Bool` 
-    whether to include the startChar and endChar. Default is true
-# Return
-  the characters between specified characters.
+This function constructs a SQL INSERT statement by extracting values for specified columns
+from a dictionary and formatting them into a valid PostgreSQL INSERT query.
+
+# Function Workflow:
+1. Iterates through the dictionary key-value pairs
+2. Filters keys to only include those present in `columnToInsert`
+3. Collects column names and their corresponding values
+4. Constructs the final SQL INSERT statement
+
+# Arguments:
+  - `table_name::String` - Name of the database table to insert into
+  - `columnToInsert::Vector{Symbol}` - List of column names to include in the INSERT statement
+  - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs for the insert
+
+# Return:
+  - A String containing the SQL INSERT statement
 
 # Example
 ```jldoctest
-julia> using Revise
-julia> using GeneralUtils
-julia> insert_data = Dict(
-      :grape => "NA",
-      :acidity => "0",
-      :tannin => "0",
-      :country => "NA",
-      :description => "NA",
-      :region => "NA",
-      :winery => "ccc",
-      :intensity => "0",
-      :sweetness => "0",
-      :tasting_notes => "NA",
-      :wine_name => "new_wine",
-      :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad",
-      :wine_type => "NA",
-      :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}",
-      :fizziness => "0",
-      :serving_temperature => "0",
-      :additional_search_term => "{NA1,NA2}")
-```
-# TODO
-  - [] update docs
+julia> using UUIDs
 
-# Signature
+# Insert a single record with specific columns
+table_name = "wine"
+columnToInsert = [:acidity, :tannin, :country, :region, :winery]
+data = Dict{Symbol, Any}(
+    :grape => "Cabernet Sauvignon",
+    :acidity => "medium",                 # using descriptive scale (low/medium/full)
+    :tannin => "medium-plus",             # common wine descriptor
+    :country => "France",
+    :description => "A rich and structured red wine with notes of blackcurrant, cedar, and subtle oak.",
+    :region => "Bordeaux",
+    :winery => "Château Margaux",
+    :intensity => "medium",               # intensity is usually low/medium/full
+    :sweetness => "dry",                  # dry/medium-dry/medium/medium-sweet/sweet
+    :tasting_notes => "Blackberry, graphite, tobacco, vanilla, and subtle earth.",
+    :wine_name => "Château Margaux Grand Cru",
+    :wine_id => "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a",  # UUID-like (valid hex)
+    :wine_type => "Red",
+    :other_attributes => Dict{String, Any}(
+        "vintage" => 2018,
+        "alcohol_percent" => 13.5,
+        "ph" => 3.6,
+        " aging_years" => 24,              # years in barrel
+        " producer_code" => "CM-GRAND"
+    ),
+    :fizziness => "still",
+    :serving_temperature => "16–18°C",
+    :additional_search_term => ["Cabernet", "Bordeaux red", "premium wine", "CabSav"]
+)
+
+julia> generateInsertSQL(table_name, columnToInsert, data)
+"INSERT INTO wine (acidity, tannin, country, region, winery) VALUES ('medium', 'medium-plus', 'France', 'Bordeaux', 'Château Margaux');"
+```
 """
-function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, 
-  insert_data::Dict{Symbol, Any})
+function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, data::Dict{Symbol, Any})
   columns = String[]
   values = String[]
 
-  for (key, value) in insert_data
+  for (key, value) in data
     if key ∈ columnToInsert
       push!(columns, string(key))
-      push!(values, "'$value'") #[] number should not wrapped in ''
+      value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+      push!(values, value_str)
     end
   end
   
@@ -89,113 +225,101 @@ function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol},
   
   return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
 end
-# function generateInsertSQL(table_name::String, insert_data::Dict{Symbol, Any})
-#   columns = String[]
-#   values = String[]
-  
-#   for (key, value) in insert_data
-#       push!(columns, string(key))
-#       if key == :other_attributes
-#           push!(values, "'$value'")
-#       else
-#           push!(values, "'$value'")
-#       end
-#   end
-  
-#   columns_str = join(columns, ", ")
-#   values_str = join(values, ", ")
-  
-#   return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
-# end
 
-  
+function generateInsertSQL(table_name::String, data::AbstractDict{String, Any})
+  columns = String[]
+  values = String[]
 
-
-
-"""
-example:
-
-insert_data = Dict(
-    :grape => "NA",
-    :acidity => "0",
-    :tannin => "0",
-    :country => "NA",
-    :description => "NA",
-    :region => "NA",
-    :winery => "ccc",
-    :intensity => "0",
-    :sweetness => "0",
-    :tasting_notes => "NA",
-    :wine_name => "new_wine",
-    :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad",
-    :wine_type => "NA",
-    :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}",
-    :fizziness => "0",
-    :serving_temperature => "0",
-    :additional_search_term => "{NA1,NA2}")
-
-id_keys is the primary key columns
-"""
-# function generateUpdateSQL(table_name::String, update_data::Dict{Symbol, Any}, id_keys::Vector{Symbol})
-#   set_clauses = String[]
-#   where_clauses = String[]
-  
-#   for (key, value) in update_data
-#       if key in id_keys
-#           push!(where_clauses, "$key = '$value'")
-#       else
-#           if key == :other_attributes
-#               push!(set_clauses, "$key = '$value'")
-#           else
-#               push!(set_clauses, "$key = '$value'")
-#           end
-#       end
-#   end
-  
-#   set_clause = join(set_clauses, ", ")
-#   where_clause = join(where_clauses, " AND ")
-  
-#   return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
-# end
-
-function generateUpdateSQL(table_name::String, columnToUpdate::Vector{Symbol}, 
-  updatedata::Dict{Symbol, Any}, id_keys::Vector{Symbol})
-  
-  set_clauses = String[]
-  where_clauses = String[]
-  
-  for (key, value) in updatedata
-      if key in id_keys
-          push!(where_clauses, "$key = '$value'")
-      else
-        if key ∈ columnToUpdate # update only specified columns
-          push!(set_clauses, "$key = '$value'")
-        end
-      end
+  for (key, value) in data
+    push!(columns, string(key))
+    value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+    push!(values, value_str)
   end
   
-  set_clause = join(set_clauses, ", ")
-  where_clause = join(where_clauses, " AND ")
-  
-  return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
+  columns_str = join(columns, ", ")
+  values_str = join(values, ", ")
+    
+    return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
 end
 
 
+# ---------------------------------------------- 100 --------------------------------------------- #
 
+""" generateUpdateSQL - Generate SQL UPDATE statement from dictionary data
 
+This function constructs a SQL UPDATE statement by updating multiple columns
+based on a primary key condition.
 
+# Arguments:
+  - `table_name::String` - Name of the database table to update
+  - `pk_column::Symbol` - The primary key column name
+  - `pk_value` - The primary key value (used in WHERE clause)
+  - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs to update
 
+# Return:
+  - A String containing the SQL UPDATE statement
 
+# Example
+```jldoctest
+julia> using UUIDs
 
+# Update multiple columns using a dictionary
+table_name = "wine"
+pk_column = :wine_id
+pk_value = "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a"
+data = Dict{Symbol, Any}(
+    :acidity => "full",
+    :tannin => "medium",
+    :country => "Italy"
+)
 
+julia> generateUpdateSQL(table_name, pk_column, pk_value, data)
+"UPDATE wine SET acidity = 'full', tannin = 'medium', country = 'Italy' WHERE wine_id = '8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a';"
+```
+"""
+function generateUpdateSQL(table_name::String, pk_column::String, pk_value, 
+                          data::AbstractDict{String, Any})
+    # Build SET clause
+    set_parts = String[]
+    for (key, value) in data
+        if key ∉ [pk_column]
+          value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+          push!(set_parts, "$(string(key)) = $value_str")
+        end
+    end
+    
+    set_clause = join(set_parts, ", ")
+    
+    # Handle primary key value
+    pk_val_str = isa(pk_value, AbstractString) ? "'$pk_value'" : "$pk_value"
+    
+    return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
+end
 
-
-
-
-
-
-
-
+function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any}, 
+                          data::AbstractDict{String, Any})
+    # Build SET clause
+    set_parts = String[]
+    for (key, value) in data
+        if key ∉ keys(pk_dict)
+            value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+            push!(set_parts, "$(string(key)) = $value_str")
+        end
+    end
+    
+    set_clause = join(set_parts, ", ")
+    
+    # Build WHERE clause for composite keys
+    where_parts = String[]
+    for (col, val) in pk_dict
+        val_str = isa(val, AbstractString) ? "'$val'" : "$val"
+        push!(where_parts, "$(string(col)) = $val_str")
+    end
+    
+    where_clause = join(where_parts, " AND ")
+    
+    return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
+end
 
 
 end   # module

src/garageS3.jl

@@ -0,0 +1,565 @@
+module garageS3
+
+export
+    LifecycleExpiration, build_lifecycle_xml,
+    GarageStorage, s3PutFile, s3GetFile, s3ListFiles, s3DeleteFile,
+    s3SetLifecycleExpiration, s3GetLifecycle, s3DeleteLifecycle,
+    s3SignUrl, s3SignUrlWeb
+
+using AWS 
+@service S3
+using AWSS3
+using Dates: Dates
+using URIs: URIs
+using OrderedCollections: OrderedDict
+using MbedTLS: digest, MD_SHA256
+using Base64: bytes2hex
+# ---------------------------------------------- 100 --------------------------------------------- #
+
+# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
+# this file use local AWS config
+
+""" Example
+storage1 = GeneralUtils.GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+  "sommpanion"
+)
+
+# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
+# use UUID as filename because GarageS3 requires unique filename for each uploaded object
+urls = GeneralUtils.s3PutFile(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
+key = GeneralUtils.s3ListFiles(storage)
+
+println("Read back: ", data)
+println("Bucket keys: ", key)
+
+# test with curl
+curl -v \
+  -H 'Host: testbucket.s3-web.yiem.cc' \
+  http://192.168.88.106:3902/users-1002.json
+
+# test with API
+data = String(GeneralUtils.s3GetFile(storage, key[1]))
+
+# test with a browser (urls.web)
+https://s3-web.mydomain.com/my_bucket_name/users-1005.json
+
+"""
+
+# ===================================================================
+# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
+# ===================================================================
+
+mutable struct SimpleCredentials
+    access_key_id::String
+    secret_key::String
+    token::String
+end
+
+mutable struct GarageConfig <: AWS.AbstractAWSConfig
+    endpoint::String
+    region::String
+    credentials::SimpleCredentials
+end
+
+# Required extensions for AWS.jl pipeline
+AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
+AWS.credentials(c::SimpleCredentials) = c
+AWS.check_credentials(c::SimpleCredentials) = c
+AWS.region(aws::GarageConfig) = aws.region
+AWS.credentials(aws::GarageConfig) = aws.credentials
+
+""" Generate the full service URL including the resource path for a Garage S3 request.
+
+# Arguments
+- `aws::GarageConfig`
+  The Garage configuration containing the endpoint.
+- `service::String`
+  The AWS service name (e.g., `"s3"`).
+- `resource::String`
+  The S3 resource path (e.g., `"/bucket/key"`).
+
+# Return
+- `String`: The full URL combining the endpoint and resource path.
+"""
+function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
+    endpoint = strip(aws.endpoint, '/')
+    resource_path = startswith(resource, '/') ? resource : "/" * resource
+    return string(endpoint, resource_path)
+end
+
+# ===================================================================
+# 2. Thread-Safe Storage Client Wrapper
+# ===================================================================
+
+""" Create a thread-safe Garage S3 storage client.
+
+# Arguments
+- `endpoint::String`
+  The Garage S3 API endpoint URL (e.g., `"https://s3-api.yiem.cc"`).
+- `key::String`
+  The Garage access key ID (created via garage-ui).
+- `secret::String`
+  The secret key corresponding to the access key.
+- `bucket::String`
+  The name of the S3 bucket to use.
+
+# Keyword Arguments
+- `region::String = "garage"`
+  The region identifier for the Garage instance.
+
+# Return
+- `GarageStorage`: A thread-safe storage client instance.
+
+# Examples
+```jldoctest
+julia> storage = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
+GarageStorage(GarageConfig(...), "sommpanion-s3")
+```
+"""
+mutable struct GarageStorage
+    config::GarageConfig
+    bucket::String
+end
+function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
+    creds = SimpleCredentials(key, secret, "")
+    config = GarageConfig(endpoint, region, creds)
+    return GarageStorage(config, bucket)
+end
+
+""" Upload an object to the Garage S3 bucket (callable struct).
+
+# Arguments
+- `key::String`
+  The object key (name) in the bucket.
+- `data::Union{String, Vector{UInt8}}`
+  The data to upload.
+
+# Return
+- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
+
+# Examples
+```jldoctest
+julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
+julia> s3PutFile1 = s3PutFile(storage1)
+julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}")
+Successfully uploaded:  test.json
+julia> downloadUrl.api
+"https://s3-api.yiem.cc/sommpanion-s3/test.json"
+julia> downloadUrl.web
+"https://s3-web.yiem.cc/sommpanion-s3/test.json"
+```
+"""
+mutable struct s3PutFile
+  storage::GarageStorage
+end
+function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}}
+  )::NamedTuple{(:api, :web), Tuple{String, String}}
+  AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
+  println("Successfully uploaded: ", key)
+
+  # object url
+  api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
+
+  # file download URL using browser
+  web = replace(api, "s3-api" => "s3-web")
+
+  return (api=api, web=web)
+end
+
+""" Upload an object to the Garage S3 bucket.
+
+# Arguments
+- `storage::GarageStorage`
+  The storage client instance.
+- `key::String`
+  The object key (name) in the bucket.
+- `data::Union{String, Vector{UInt8}}`
+  The data to upload.
+
+# Return
+- `NamedTuple{(:api, :web)}`: A tuple with `api` and `web` URLs.
+
+# Examples
+```julia
+julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}")
+Successfully uploaded:  test.json
+julia> result.api
+"https://s3-api.yiem.cc/sommpanion-s3/test.json"
+julia> result.web
+"https://s3-web.yiem.cc/sommpanion-s3/test.json"
+```
+"""
+function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
+  )::NamedTuple{(:api, :web), Tuple{String, String}}
+  AWSS3.s3_put(storage.config, storage.bucket, key, data)
+  println("Successfully uploaded: ", key)
+
+  # object url
+  api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
+
+  # file download URL using browser
+  web = replace(api, "s3-api" => "s3-web")
+
+  return (api=api, web=web)
+end
+
+""" Download an object from the Garage S3 bucket.
+
+# Arguments
+- `storage::GarageStorage`
+  The storage client instance.
+- `key::String`
+  The object key (name) in the bucket.
+
+# Return
+- `Vector{UInt8}`: The raw bytes of the downloaded object.
+
+# Examples
+```julia
+julia> data = String(s3GetFile(storage, "test.json"))
+"\"{\\\"status\\\": \\\"active\\\"}\""
+```
+"""
+function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
+    try
+        return AWSS3.s3_get(storage.config, storage.bucket, key; raw=true)
+    catch
+        return nothing
+    end
+end
+
+""" List all object keys in the Garage S3 bucket.
+
+# Arguments
+- `storage::GarageStorage`
+  The storage client instance.
+
+# Return
+- `Vector{String}`: A list of object keys (file names) in the bucket.
+
+# Examples
+```julia
+julia> keys = s3ListFiles(storage)
+3-element Vector{String}:
+ "test1.json"
+ "test2.json"
+ "users-1002.json"
+```
+"""
+function s3ListFiles(storage::GarageStorage)
+    # Use delimiter="" to get all individual keys (no grouping by "directory")
+    objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="")
+    return [obj["Key"] for obj in objects]
+end
+
+""" Delete an object from the Garage S3 bucket.
+
+# Arguments
+- `storage::GarageStorage`
+  The storage client instance.
+- `key::String`
+  The object key (name) to delete.
+
+# Examples
+```julia
+julia> s3DeleteFile(storage, "test.json")
+```
+"""
+function s3DeleteFile(storage::GarageStorage, key::String)
+    AWSS3.s3_delete(storage.config, storage.bucket, key)
+end
+
+"""
+    s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String
+
+Generate a pre-signed URL for an object in the Garage S3 bucket.
+
+A pre-signed URL grants temporary access without AWS credentials. Signature is computed
+using AWS Signature Version 4 (HMAC-SHA256).
+
+# Arguments
+- `storage::GarageStorage`: Storage client from `GarageStorage(endpoint, key, secret, bucket)`.
+- `key::String`: Object key. Supports nested paths like `"folder/subfolder/file.txt"`.
+- `seconds::Int`: URL validity in seconds. Default: 3600.
+
+# Keyword Arguments
+- `verb::String = "GET"`: HTTP method (`"GET"` downloads, `"PUT"` uploads).
+- `content_type::String = "application/octet-stream"`: Required `Content-Type` header.
+  For PUT, set to the file's MIME type (`"image/png"`, `"application/json"`, etc.).
+- `signature_version::String = "v4"`: Always use `"v4"` (v2 is deprecated).
+
+# Return
+`String`: The pre-signed URL.
+
+# Examples
+
+```julia
+using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile
+
+storage = GarageStorage(
+  "https://s3-api.yiem.cc",
+  "GKb080154a2e5b19100b1b2c6e",
+  "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+  "sommpanion"
+)
+
+# Upload — PUT with curl
+url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg")
+# curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' <url>
+
+# Download — GET with curl
+url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET")
+# curl <url>  → returns file content
+
+# Upload JSON
+url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json")
+# curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' <url>
+
+# Delete
+url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE")
+# curl -X DELETE url
+
+# Multiple uploads (chat app)
+urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300;
+                     verb="PUT", content_type="image/png")
+        for _ in 1:10]
+
+# Signature differs by verb and content_type
+url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET")
+url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain")
+```
+
+# Notes
+- URLs are valid until `seconds` expires; they can be used **multiple times** (not one-time).
+- Server clocks must be within ~5 minutes of the Garage server (UTC time in signature).
+- PUT uploads require the client's `Content-Type` to match `content_type`.
+- For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`.
+"""
+function s3SignUrl(
+    storage::GarageStorage,
+    key::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    signature_version::String="v4",
+)::String
+    config = storage.config
+    creds = config.credentials
+    bucket = storage.bucket
+    region = config.region
+    endpoint = strip(config.endpoint, '/')
+    scheme = "AWS4"
+    algorithm = "HMAC-SHA256"
+    terminator = "aws4_request"
+    service = "s3"
+
+    now_datetime = Dates.now(Dates.UTC)
+    datetime_stamp = Dates.format(now_datetime, "YYYYmmddTHHMMSS\\Z")
+    date_stamp = Dates.format(now_datetime, "YYYYmmdd")
+
+    scope = "$date_stamp/$region/$service/$terminator"
+
+    headers = OrderedDict{String,String}("Host" => URIs.URI(endpoint).host)
+    if verb == "PUT"
+        headers["Content-Type"] = content_type
+    end
+    sort!(headers; by=name -> lowercase(name))
+    canonical_header_names = join(map(name -> lowercase(name), collect(keys(headers))), ";")
+
+    query = OrderedDict{String,String}(
+        "X-Amz-Expires" => string(seconds),
+        "X-Amz-Algorithm" => "$scheme-$algorithm",
+        "X-Amz-Credential" => "$(creds.access_key_id)/$scope",
+        "X-Amz-Date" => datetime_stamp,
+        "X-Amz-SignedHeaders" => canonical_header_names,
+    )
+    if !isempty(creds.token)
+        query["X-Amz-Security-Token"] = creds.token
+    end
+
+    canonical_headers = join(
+        map(
+            header -> "$(lowercase(header.first)):$(lowercase(header.second))\n",
+            collect(headers),
+        ),
+    )
+
+    resource_path = "/$bucket/$key"
+
+    # Build canonical query string: sort by key=value pair lexicographically,
+    # URL-encoding each key and value (matching Garage's uri_encode)
+    sorted_q = sort(
+        [URIs.escapeuri(k) * "=" * URIs.escapeuri(v) for (k, v) in pairs(query)],
+        by=lowercase,
+    )
+    canonical_query = join(sorted_q, "&")
+
+    canonical_request = string(
+        "$verb\n",
+        resource_path,
+        "\n",
+        canonical_query,
+        "\n",
+        canonical_headers,
+        "\n",
+        canonical_header_names,
+        "\n",
+        "UNSIGNED-PAYLOAD",
+    )
+
+    string_to_sign = string(
+        "$scheme-$algorithm\n",
+        "$datetime_stamp\n",
+        "$scope\n",
+        bytes2hex(digest(MD_SHA256, canonical_request)),
+    )
+
+    key_secret_bytes = Vector{UInt8}(string(scheme, creds.secret_key))
+    key_date = digest(MD_SHA256, date_stamp, key_secret_bytes)
+    key_region = digest(MD_SHA256, region, key_date)
+    key_service = digest(MD_SHA256, service, key_region)
+    key_signing = digest(MD_SHA256, terminator, key_service)
+    signature = digest(MD_SHA256, string_to_sign, key_signing)
+
+    query["X-Amz-Signature"] = bytes2hex(signature)
+
+    query_string = URIs.escapeuri(query)
+    return "$(endpoint)/$bucket/$key?$query_string"
+end
+
+""" Generate a pre-signed URL using the Garage S3 web endpoint (s3-web).
+
+Useful for direct browser access — URLs look like
+`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...`
+
+Arguments and keyword arguments are the same as `s3SignUrl`.
+"""
+function s3SignUrlWeb(
+    storage::GarageStorage,
+    key::String,
+    seconds::Int=3600;
+    verb::String="GET",
+    content_type::String="application/octet-stream",
+    signature_version::String="v4",
+)::String
+    web_endpoint = replace(storage.config.endpoint, "s3-api" => "s3-web")
+    web_creds = SimpleCredentials(
+        storage.config.credentials.access_key_id,
+        storage.config.credentials.secret_key,
+        storage.config.credentials.token,
+    )
+    web_config = GarageConfig(web_endpoint, storage.config.region, web_creds)
+    web_storage = GarageStorage(web_config, storage.bucket)
+    return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version)
+end
+
+# ===================================================================
+# 3. Bucket Lifecycle Configuration
+# ===================================================================
+
+""" Lifecycle expiration configuration for a bucket.
+
+# Fields
+- `prefix::String`: Object key prefix this rule applies to.
+- `days::Int`: Number of days after object creation to expire/delete.
+- `enabled::Bool`: Whether the rule is active (default: `true`).
+- `id::String`: Unique identifier for the rule (default: auto-generated).
+"""
+mutable struct LifecycleExpiration
+    prefix::String
+    days::Int
+    enabled::Bool
+    id::String
+end
+function LifecycleExpiration(prefix::String, days::Int; enabled::Bool=true, id::String="")
+    rule_id = isempty(id) ? "expire-" * replace(prefix, "/" => "_") * "-" * string(days) * "d" : id
+    return LifecycleExpiration(prefix, days, enabled, rule_id)
+end
+
+""" Build the XML body for a PutBucketLifecycleConfiguration request.
+
+Supports a single rule with expiration by prefix.
+"""
+function build_lifecycle_xml(rule::LifecycleExpiration)::String
+    status = rule.enabled ? "Enabled" : "Disabled"
+    return """<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+  <Rule>
+    <ID>$(rule.id)</ID>
+    <Filter>
+      <Prefix>$(rule.prefix)</Prefix>
+    </Filter>
+    <Status>$status</Status>
+    <Expiration>
+      <Days>$(rule.days)</Days>
+    </Expiration>
+  </Rule>
+</LifecycleConfiguration>"""
+end
+
+""" Set bucket lifecycle expiration rule.
+
+# Arguments
+- `storage::GarageStorage`: The storage client instance.
+- `rule::LifecycleExpiration`: The lifecycle expiration rule.
+
+# Returns
+- `Dict{String, Any}`: Parsed response from the server.
+
+# Example
+```julia
+rule = LifecycleExpiration("uploads/", 30)
+s3SetLifecycleExpiration(storage, rule)
+```
+"""
+function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration)
+    xml_body = build_lifecycle_xml(rule)
+    args = Dict{String,Any}("body" => xml_body)
+    return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config)
+end
+
+""" Get the bucket lifecycle configuration.
+
+# Arguments
+- `storage::GarageStorage`: The storage client instance.
+
+# Returns
+- `Dict{String, Any}`: Parsed lifecycle configuration, or `nothing` if no lifecycle is configured.
+
+# Example
+```julia
+config = s3GetLifecycle(storage)
+```
+"""
+function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
+    try
+        return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config)
+    catch e
+        if isa(e, AWS.AWSException) && e.code == "NoSuchLifecycleConfiguration"
+            return nothing
+        end
+        rethrow(e)
+    end
+end
+
+""" Delete the bucket lifecycle configuration.
+
+# Arguments
+- `storage::GarageStorage`: The storage client instance.
+
+# Returns
+- `Dict{String, Any}`: Parsed response from the server.
+
+# Example
+```julia
+s3DeleteLifecycle(storage)
+```
+"""
+function s3DeleteLifecycle(storage::GarageStorage)
+    return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config)
+end
+
+end # module GarageS3

src/util.jl

@@ -1,43 +1,46 @@
 module util
   
 export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, replaceDictKeys,
-        findMatchingDictKey, textToDict, randstring, randstrings, timeout,
+        findMatchingDictKey, randstring, randstrings, timeout,
         dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString, 
-        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, 
-        dictToString_numbering, extract_triple_backtick_text, 
-        countGivenWords, remove_french_accents, detect_keyword, extractTextBetweenCharacter,
-        extractTextBetweenString, 
+        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
+        dictToString_numbering, extract_triple_backtick_text, eventSink,
+        countGivenWords, remove_french_accents, removestring,
+        extractTextBetweenCharacter, extractTextBetweenString, 
         convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
 
-using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, NATS, Dates
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-""" Compute time different between start time and stop time in a given unit. 
- Unit can be "milliseconds", "seconds", "minutes", "hours".
+""" Computes the time difference between two `DateTime` values and returns the
+result in a specified unit: milliseconds, seconds, minutes, or hours.
 
 # Arguments
-  - `starttime::DateTime` 
-    start time
-  - `stoptime::DateTime`  
-    stop time
-  - `unit::String`  
-    unit of time difference
+- `starttime::DateTime`
+  The starting `DateTime` value.
+- `stoptime::DateTime`
+  The ending `DateTime` value.
+- `unit::String`
+  The unit for the result. Must be one of: `"milliseconds"`, `"seconds"`,
+  `"minutes"`, `"hours"`.
 
 # Return
-  - time difference in given unit
+- `Integer`: The time difference converted to the specified unit.
 
-# Example
+# Notes
+- The function computes `stoptime - starttime` and converts the result to the
+  requested unit using integer division.
+- Errors with `ArgumentError` if an invalid unit is specified.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils, Dates
 julia> a = Dates.now()
-julia> b = a + Dates.Day(5) # add 5 days
-julia> GeneralUtils.timedifference(a, b, "hours")
+julia> b = a + Dates.Day(5)
+julia> timedifference(a, b, "hours")
 120
 ```
-
-# Signature
 """
 function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer
   diff = stoptime - starttime
@@ -184,21 +187,27 @@ end
 
 
 
-""" Get uuid4 with snake case
-  
-# Return
-  - `uuid4::String`
-    uuid4 with snake case
+""" Generates a UUID4 (version 4) identifier and converts it to snake case by
+replacing hyphens with underscores.
 
-# Example
+# Arguments
+- This function takes no arguments.
+
+# Return
+- `String`: A UUID4 string with underscores instead of hyphens (e.g.,
+  `"0f6e4f_568c_4df4_8c79_1d7a58072f4a"`).
+
+# Notes
+- Uses the `uuid4()` function from the UUIDs standard library to generate a
+  random UUID.
+- The underscore character replaces all hyphens in the UUID string.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils
-julia> GeneralUtils.uuid4snakecase()
+julia> uuid4snakecase()
 "0f6e4f_568c_4df4_8c79_1d7a58072f4a"
 ```
-
-# Signature
 """
 function uuid4snakecase()::String
   _id = string(uuid4())
@@ -207,32 +216,37 @@ function uuid4snakecase()::String
 end
 
 
-""" Replace a dictionary key with the new key
+""" Replaces keys in a dictionary according to a mapping, returning a new
+dictionary with updated keys while preserving the original values.
 
 # Arguments
-  - `d::Dict` 
-    The input dictionary that you want to modify
-  - `replacementMap::Dict`
-    A dictionary that maps old keys to new keys
+- `d::Dict`
+  The input dictionary to modify.
+- `replacementMap::Dict`
+  A dictionary mapping old keys to new keys. Keys not present in this map are
+  left unchanged.
 
 # Return
-  - `newDict::Dict`
-    new dictionary with the replaced keys
+- `Dict`: A new dictionary with replaced keys. Values are preserved from the
+  original dictionary.
 
-# Example
+# Notes
+- The function creates a new dictionary rather than modifying the input in
+  place.
+- Keys not found in `replacementMap` are copied to the result with their
+  original keys unchanged.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils
 julia> d = Dict(:a => 1, :b => 2, :c => 3)
 julia> replacement_map = Dict(:a => :x, :b => :y)
-julia> new_dict = GeneralUtils.replaceDictKeys(d, replacement_map)
+julia> replaceDictKeys(d, replacement_map)
 Dict{Any, Any} with 3 entries:
   :y => 2
   :c => 3
   :x => 1
 ```
-
-# Signature
 """
 function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
   newDict = Dict()
@@ -244,102 +258,6 @@ function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
 end
 
 
-""" Convert text into a dictionary with a given keywords. This function use keywords to slice
-  a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text.
-  The left most string which has no keyword will be discarded. WARNING, ordering is important
-
-# Arguments
-  - `text::String`
-    A text to be converted.
-  - `keywords::Vector{String}`
-    A list of keywords to be used to slice the text. 
-    These keywords also be the resulting dict keys.
-# Keyword Arguments
-  - `rightmarker::String`
-    A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", 
-    the function will search for "plan:" otherwise the function will search for "plan".
-    The marker will not be in the resulting dict keys.
-  - `symbolkey::Bool`
-    If true, resulting dict's key will be Symbols, otherwise string.
-  - `lowercasekey::Bool`
-    set resulting dict's key to be lowercase
-
-# Return
-  - `d::OrderedDict`
-
-# Example
-```jldoctest
-julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep"
-julia> sample_keywords = ["thought", "plan", "action"]
-julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true)
-julia> println(resultdict)
-OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", 
-                                        :plan => "wake up and going out", 
-                                        :action => "1. wake up 2. eat 3. sleep")
-```
-
-# Signature
-"""
-function textToDict(text::String, detectKeywords::Vector{String}; 
-    dictKey::Union{Vector{String}, Nothing}=nothing, 
-    symbolkey::Bool=false, lowercasekey::Bool=false
-    )::OrderedDict
-
-  # make sure this function detect variation of a work e.g. agent, Agent, AGENT
-  kw = []
-  # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
-  for keyword in detectKeywords
-    detected = detect_keyword(keyword, text)
-    if detected !== nothing
-      push!(kw, detected)
-    else
-      error("Keyword $keyword not found in text.")
-    end
-  end
-
-  od1, od2 =
-  if symbolkey
-    OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}()
-  else
-    OrderedDict{String, Any}(), OrderedDict{String, Any}()
-  end
-
-  remainingtext = text
-  dictKey_ = reverse(dictKey)
-  
-  # process text from back to front
-  rkw = reverse(kw)
-  for (i,keyword) in enumerate(rkw)
-    # Find the position of the keyword in the text
-    keywordidx = findlast(keyword, remainingtext)
-    dKey = dictKey_[i]
-    
-    if keywordidx !== nothing
-      substr = remainingtext[keywordidx[end]+1:end]
-      str = string(strip(substr))  # Removes both leading and trailing whitespace.
-      _key = lowercasekey == true ? lowercase(dKey) : dKey
-      key = symbolkey == true ? Symbol(_key) : _key
-      od1[key] = str
-      remainingtext = remainingtext[1:keywordidx[1]-1]
-    else
-      error("""keyword "$keyword" not found in the provided text: $text </end of error note>""")
-    end
-  end
-
-  # correct the order
-  ks = reverse([i for i in keys(od1)])
-  for k in ks
-    k = symbolkey == true ? Symbol(k) : k
-    od2[k] = od1[k]
-  end
-  
-  return od2
-end
-
-
-
-
-
 """ Generate a random string
 
 # Arguments
@@ -390,36 +308,43 @@ end
 
 
 
-""" Execute a function with timer.
+""" Executes a function with a timeout mechanism. If the function does not
+complete within the specified time, it is interrupted and a timeout message
+is returned.
 
 # Arguments
-  - `f::Function`
-      a function to run
-  - `timeoutwindow::Integer``
-      timeout in seconds
+- `f::Function`
+  The function to execute.
+- `timeoutwindow::Integer`
+  The timeout duration in seconds.
 
-# Keyword Argument
-  - `fargs`
-    arguments for the function
-  - `timeoutmsg::String`
-    time out message
+# Keyword Arguments
+- `fargs`
+  Arguments to pass to the function `f`. If `nothing`, the function is called
+  without arguments.
+- `timeoutmsg::String`
+  The message to return if the function times out. Defaults to `"task timed out"`.
 
 # Return
-  - task result otherwise timeout message
+- The result of the function if it completes within the timeout, otherwise the
+  `timeoutmsg` string.
 
-# Example
-  ```jldoctest
+# Notes
+- Uses Julia's `@task`, `schedule`, and `Timer` to implement non-blocking
+  execution with interruption via `Base.throwto`.
+- Errors with `InterruptException` if the function exceeds the timeout.
+
+# Examples
+```jldoctest
 julia> function testfunc(x)
-          sleep(x)
-          return "task done"
-        end
+           sleep(x)
+           return "task done"
+       end
 julia> result = timeout(testfunc, 10; fargs=20)
 "task timed out"
 julia> result = timeout(testfunc, 20; fargs=10)
 "task done"
 ```
-
-#  Signature
 """
 function timeout(f::Function, timeoutwindow::Integer; fargs=nothing, timeoutmsg="task timed out")
   tsk = @task f(fargs)
@@ -436,23 +361,26 @@ end
 
 
 
-""" Convert a dataframe into CSV.
+""" Converts a DataFrame to a CSV string representation using the CSV.jl package.
 
 # Arguments
-  - `df::DataFrame`
-    A connection object to Postgres database
-    
-# Return
-  - `result::String`
+- `df::DataFrame`
+  The DataFrame to convert to CSV format.
 
-# Example
+# Return
+- `String`: The DataFrame contents as a CSV-formatted string.
+
+# Notes
+- Uses `CSV.write` with an `IOBuffer` to capture the output as a string.
+- The returned string contains the full CSV representation including headers.
+
+# Examples
 ```jldoctest
 julia> using DataFrames, GeneralUtils
 julia> df = DataFrame(A=1:3, B=5:7, fixed=1)
-julia> result = GeneralUtils.dataframeToCSV(df)
+julia> dataframeToCSV(df)
+"1,5,1\n2,6,1\n3,7,1\n"
 ```
-
-# Signature
 """
 function dataframeToCSV(df::DataFrame)
   # Create an IOBuffer to capture the output
@@ -474,7 +402,7 @@ end
 
 #  Example
   ```jldoctest
-  julia> using DataFrames, JSON3, GeneralUtils
+  julia> using DataFrames, GeneralUtils
   julia> df = DataFrame(A = [1, 2, 3], B = ["apple", "banana", "cherry"])
   julia> vectorDict = GeneralUtils.dfToVectorDict(df)
   [Dict{String, Any}("B" => "apple", "A" => 1), 
@@ -512,7 +440,7 @@ end
 
 # Example
   ```jldoctest
-  julia> using GeneralUtils, Dates, JSON3, UUIDs
+  julia> using GeneralUtils, Dates, UUIDs
   julia> vecDict = [Dict("a" => i) for i in 1:10]
   julia> d = GeneralUtils.disintegrate_vectorDict(vecDict, 3)
   julia> println(d[:data])
@@ -522,8 +450,6 @@ end
   3 => [Dict("a"=>7), Dict("a"=>8), Dict("a"=>9)]
   4 => [Dict("a"=>10)]
   ```
-
-# Signature
 """
 function disintegrate_vectorDict(data::Vector, partsize::Integer
   )
@@ -569,8 +495,6 @@ end
   julia> getDataFrameValue(df[1, :], :name)
   "Alice"
   ```
-
-# Signature
 """
 getDataFrameValue(row::DataFrameRow, key::Symbol) = row.:($key)
 
@@ -639,8 +563,6 @@ end
   julia> dfToString(df)
   "1) name: Alice, age: 25\n2) name: Bob, age: 30"
   ```
-
-# Signature
 """
 function dfToString(df::DataFrame)
   dfstr = ""
@@ -664,7 +586,7 @@ end
 
 # Example
   ```jldoctest
-  julia> using DataFrames
+  julia> using DataFrames, GeneralUtils
 
   julia> df = DataFrame(name=["Alice", "Bob"], age=[25, 30])
   2×2 DataFrame
@@ -674,13 +596,11 @@ end
    │ 1   │ Alice   25    
    │ 2   │ Bob     30    
 
-  julia> dataframe_to_json_list(df)
+  julia> GeneralUtils.dataframe_to_json_list(df)
   2-element Vector{String}:
   "{\"name\":\"Alice\",\"age\":25}"
   "{\"name\":\"Bob\",\"age\":30}"
   ```
-
-# Signature
 """
 function dataframe_to_json_list(df::DataFrame)::Vector{String}
   json_list = []
@@ -714,8 +634,6 @@ end
   julia> dict_to_string(od)
   "1) name: Alice, 2) age: 25"
   ```
-
-# Signature
 """
 function dictToString(od::T) where {T<:AbstractDict}
   items = []
@@ -784,152 +702,6 @@ function cuttext(range, text)
   end
 end
 
-"""
-    detect_keyword(keywords::AbstractVector{String}, text::String; mode::Union{String, Nothing}=nothing, delimiter::AbstractVector=[' ', '\n', '.']) -> Dict{String, Integer}
-
-Detects and counts occurrences of multiple keywords in the text in different case variations (lowercase, uppercase first letter, or all uppercase).
-
-# Arguments
-- `keywords::AbstractVector{String}` Vector of keywords to search for
-- `text::String` The text to search in
-
-# Keyword Arguments
-- `mode::Union{String, Nothing}` When set to "individual", only counts matches that are individual words (default: nothing)
-- `delimiter::AbstractVector` Characters used to determine word boundaries when mode="individual" (default: [' ', '\n', '.'])
-
-# Returns
-- `Dict{String, Integer}` Returns a dictionary mapping each keyword to its count in the text (0 if not found)
-
-# Examples
-  ```jldoctest
-  julia> detect_keyword(["test", "example"], "This is a Test EXAMPLE")
-  Dict{String, Integer}("test" => 1, "example" => 1)
-
-  julia> detect_keyword(["cat"], "cats and category", mode="individual")
-  Dict{String, Integer}("cat" => 0)
-
-  julia> detect_keyword(["error"], "No ERRORS found!")
-  Dict{String, Integer}("error" => 1)
-  ```
-  
-# Signature
-"""
-# function detect_keyword(keywords::T1, text::String; 
-#   mode::Union{String, Nothing}=nothing, delimiter::T2=[' ', '\n', '.']
-#   )::Dict{String, Integer} where {T1<:AbstractVector, T2<:AbstractVector}
-#   # Initialize dictionary to store keyword counts
-#   kwdict = Dict{String, Integer}()
-#   for i in keywords
-#     kwdict[i] = 0
-#   end
-  
-#   startindex = 1
-#   # Iterate through each keyword and search for matches in text
-#   for kw in keywords
-#     # Check each possible starting position in the text
-#     for startindex in 1:1:length(text)
-#       # Get the window range for current keyword at current position
-#       wordwindows = wordwindow(kw, startindex)
-#       # Extract the text slice for comparison
-#       cuttexts = cuttext(wordwindows, text)
-#       if cuttexts !== nothing
-#         # Try to detect keyword in current text slice
-#         detected_kw = detect_keyword(kw, cuttexts)
-#         if detected_kw !== nothing && mode === nothing
-#           # Increment count if keyword found and no mode restrictions
-#           kwdict[kw] +=1
-#         elseif detected_kw !== nothing && mode === "individual"
-#           # For individual word mode, check word boundaries
-#           # Check if character before keyword is a delimiter or start of text
-#           checkbefore = 
-#             if wordwindows.start > 1 && 
-#               text[wordwindows.start-1] ∈ delimiter
-#               true
-#             elseif wordwindows.start == 1
-#               true
-#             else
-#               false
-#             end
-
-#           # Check if character after keyword is a delimiter or end of text
-#           checkafter =
-#             if wordwindows.stop < length(text) && 
-#               text[wordwindows.stop+1] ∈ delimiter
-#               true
-#             elseif wordwindows.stop == length(text)
-#               true
-#             else
-#               false
-#             end
-#           # Only count keyword if it's a complete word
-#           if checkbefore && checkafter
-#             kwdict[kw] +=1
-#           end
-#         end
-#       end
-#     end
-#   end
-#   return kwdict
-# end
-
-
-function detect_keyword(keywords::T, text::String)::Dict{String, Integer} where {T<:AbstractVector}
-  kw = Dict{String, Integer}()
-  splittext = string.(split(text, " "))
-  # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
-  for keyword in keywords
-    ws = detect_keyword.(keyword, splittext)
-    total = sum(issomething.(ws))
-    if total != 0
-      kw[keyword] = total
-    else
-      kw[keyword] = 0
-    end
-  end
-  return kw
-end
-
-
-"""
-    detect_keyword(keyword::String, text::String) -> Union{Nothing, String}
-
-Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase).
-
-# Arguments:
-- `keyword::String` The keyword to search for
-- `text::String` The text to search in
-
-# Returns:
-- `Union{Nothing, String}` Returns the matched keyword variation if found, otherwise returns nothing
-
-# Examples:
-  ```jldoctest
-  julia> detect_keyword("test", "This is a Test case")
-  "Test"
-
-  julia> detect_keyword("error", "NO ERRORS FOUND")
-  "ERRORS"
-
-  julia> detect_keyword("missing", "complete data")
-  nothing
-  ```
-
-# Signature
-"""
-function detect_keyword(keyword::String, text::String)::Union{Nothing, String}
-  # Define the keyword variations to search for
-  keyword_variations = [keyword, uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)]
-  
-  # Check if any of the keyword variations are in the text
-  for variation in keyword_variations
-      if occursin(variation, text)
-          return variation
-      end
-  end
-  
-  # Return nothing if no variation is found
-  return nothing
-end
 
 
 """
@@ -975,27 +747,39 @@ end
 
 
 
-"""
-    remove_french_accents(text::String) -> String
+""" Remove French accents from the given text.
 
-Remove French accents from the given text.
+The function replaces accented French characters with their non-accented
+counterparts using a dictionary mapping. Supported accented characters
+include: à, â, ä, á, é, è, ê, ë, î, ï, í, ñ, ô, ö, ò, ó, ù, û, ü, ÿ, ç,
+and their uppercase variants. The apostrophe character `’` is removed
+completely.
 
 # Arguments
-- `text::String` The input string containing French accents.
+- `text::AbstractString`
+  The input string containing French accented characters.
 
-# Returns
-- `String` The input string with all French accents removed.
+# Return
+- `AbstractString`: A new string with all French accents replaced by their
+  non-accented equivalents.
+
+# Notes
+- The function creates a character list and replaces each accented character
+  according to an internal dictionary mapping.
+- Does **not** mutate the input; it allocates a new string.
 
 # Examples
-  ```jldoctest
-  julia> remove_french_accents("Café")
-  "Cafe"
+```jldoctest
+julia> using GeneralUtils
+julia> remove_french_accents("Café")
+"Cafe"
 
-  julia> remove_french_accents("L'été est beau.")
-  "L'ete est beau."
-  ```
+julia> remove_french_accents("L'été est beau.")
+"L'ete est beau."
 
-# Signature
+julia> remove_french_accents("Noël, naïve, François")
+"Noel, naive, Francois"
+```
 """
 function remove_french_accents(text::AbstractString)::AbstractString
   textcharlist = [i for i in text]
@@ -1355,9 +1139,138 @@ function nonRecentElementsIndex(vectorlength::Integer, n::Integer)
   return 1:(vectorlength-n)
 end
 
+""" Remove specified substrings from text.
+
+Removes all occurrences of each string in `removelist` from the input text
+by repeatedly replacing them with empty strings until none remain.
+
+# Arguments
+- `text::String`
+  The input string to modify.
+- `removelist::Vector{String}`
+  A vector of substrings to remove from the text.
+
+# Return
+- `String`: The text with all specified substrings removed.
+
+# Examples
+```jldoctest
+julia> using GeneralUtils
+julia> removestring("hello world", ["l", " "])
+"heoword"
+julia> removestring("foo bar baz", ["bar", " "])
+"foobaz"
+```
+"""
+function removestring(text::String, removelist::Vector{String})::String
+  for i in removelist
+    while occursin(i, text)
+      text = replace(text, i => "")
+    end
+  end
+
+  return string(text)
+end
 
 
+"""
+Create an event sink for logging messages and/or publishing to NATS or printing to console. File logging defaults to false.
 
+# Fields
+- `natsConn`: NATS connection, or `nothing` for file-only sink
+- `topic`: NATS topic to publish to, or `nothing` for file-only sink
+- `service_name`: Optional prefix added to log entries
+- `log_file`: Path to the log file
+- `max_log_entries`: Maximum number of entries to keep in the log file
+
+# Callable interface
+```julia
+    (es::eventSink)(msg::String; log=false, publish=true, console=true) -> Nothing
+```
+
+Three independent outputs — each defaults to `true` except `log`:
+- `console` (default `true`) → println to stdout
+- `publish` (default `true`) → publish to NATS topic (no-op if no connection)
+- `log` (default `false`) → append to `log_file`
+
+# Examples
+```julia
+using NATS, Dates
+
+conn = NATS.connect("nats.yiem.cc")
+
+# ——— With NATS (console + NATS by default) ———
+es = eventSink(natsConn=conn, topic="agent.events", service_name="agent1")
+es("starting up")                 # console + NATS
+es("logged"; log=true)            # console + file + NATS
+es("quiet"; console=false)        # NATS only
+es("file+nats"; log=true, console=false)  # file + NATS
+es("console-only"; publish=false, log=false)  # console only
+
+# ——— File-only sink (no NATS connection) ———
+es2 = eventSink(log_file="./log/eventSink2.md", service_name="worker")
+es2("ready")                      # console only
+es2("file-logged"; log=true)       # console + file
+es2("file-only"; log=true, console=false)  # file only
+es2("silent"; console=false)               # nothing (no NATS, log=false, console=false)
+
+NATS.drain(conn)
+```
+"""
+struct eventSink
+  natsConn::Union{NATS.Connection, Nothing}
+  topic::Union{String, Nothing}
+  service_name::String
+  log_file::String
+  max_log_entries::Int
+end
+eventSink(; natsConn=nothing, topic=nothing, service_name="", log_file="./log/eventSink.md", max_log_entries=100) =
+  eventSink(natsConn, topic, service_name, log_file, max_log_entries)
+function (es::eventSink)(msg::String; log=false, publish=true, console=true)
+    timestamp = Dates.format(now(), "yyyy-mm-ddTHH:MM:SS.sss")
+    svc_prefix = isempty(es.service_name) ? "" : "[$(es.service_name)] "
+    entry = "- [$timestamp] $svc_prefix$msg\n"
+    if console
+        println("[$timestamp] $svc_prefix$msg")
+    end
+    if log
+        log_dir = dirname(es.log_file)
+        mkpath(log_dir)
+        open(es.log_file, "a") do f
+            write(f, entry)
+        end
+        # Rotate after write — keep only last max_log_entries entries
+        content = read(es.log_file, String)
+        entries = split(content, r"- \[", keepempty=false)
+        if length(entries) > es.max_log_entries
+            rotate!(es)
+        end
+    end
+    if publish && es.topic !== nothing && es.natsConn !== nothing
+        try
+            NATS.publish(es.natsConn, es.topic, entry)
+        catch e
+            @warn "eventSink NATS publish failed" exception=e
+        end
+    end
+end
+
+"""
+    rotate!(es::eventSink) -> Nothing
+
+Rewrite the log file keeping only the last `max_log_entries` entries.
+Call after writing when the entry count reaches the limit.
+"""
+function rotate!(es::eventSink)
+    content = read(es.log_file, String)
+    entries = split(content, r"- \[", keepempty=false)
+    open(es.log_file, "w") do f
+        write(f, "- [")
+        for e in entries[2:end]
+            write(f, "- [", e)
+        end
+    end
+end
 
 
 

test/etc.jl

@@ -1,7 +1,130 @@
-python -> pandas -> dataframe -> csv
-                
+curl -X PUT -H "Content-Type: application/json" \
+  -d '{"test": "data"}' \
+  "https://s3-api.yiem.cc/sommpanion/pcimage.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T103745Z&X-Amz-Expires=300&X-Amz-Security-Token=&X-Amz-SignedHeaders=host&X-Amz-Signature=0e195a48f824a2f8ffcb06461396cd6a8bddd5b61d93e9006f947ae10ecf525f"
+
+
+
+
+
+
+
+
+
+curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
+  -H 'Host: s3-api.yiem.cc' \
+  -H 'Content-Type: image/jpeg' \
+  "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T030315Z&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=1107a31f36f8b8ceb9663d69b9a7be7733df14d019e700e750944cbdad50df5d"
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+read
+./garage.toml
+./Caddyfile
+./src/garageS3.jl
+
+
+
+julia> url = GeneralUtils.s3_sign_url(storage1, "photo.jpg", 36000; verb="PUT", content_type="image/jpeg")
+"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T145004Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=92cd634dbb0f66db4b8b039ef5939c138203de260e8342fe27ef78979d90a824"
+
+---
+
+
+
+
+
+
+
+
+
+
+
+ton@server12vm-dev:~/docker-apps/sommpanion/etc/AWS.jl$ 
+curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
+  -H 'Host: s3-api.yiem.cc' \
+  -H 'Content-Type: image/jpeg' \
+  "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02"
+
+Note: Unnecessary use of -X or --request, PUT is already inferred.
+*   Trying 110.77.160.154:443...
+* Connected to s3-api.yiem.cc (110.77.160.154) port 443 (#0)
+* ALPN: offers h2,http/1.1
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+*  CAfile: /etc/ssl/certs/ca-certificates.crt
+*  CApath: /etc/ssl/certs
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+* SSL connection using TLSv1.3 / TLS_CHACHA20_POLY1305_SHA256
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=s3-api.yiem.cc
+*  start date: Aug 27 04:27:02 2026 GMT
+*  expire date: Nov 25 04:27:01 2026 GMT
+*  subjectAltName: host "s3-api.yiem.cc" matched cert's "s3-api.yiem.cc"
+*  issuer: C=US; O=Let's Encrypt; CN=YE2
+*  SSL certificate verify ok.
+* using HTTP/2
+* h2h3 [:method: PUT]
+* h2h3 [:path: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02]
+* h2h3 [:scheme: https]
+* h2h3 [:authority: s3-api.yiem.cc]
+* h2h3 [user-agent: curl/7.88.1]
+* h2h3 [accept: */*]
+* h2h3 [content-type: image/jpeg]
+* h2h3 [content-length: 439490]
+* Using Stream ID: 1 (easy handle 0x559a08824c80)
+> PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02 HTTP/2
+> Host: s3-api.yiem.cc
+> user-agent: curl/7.88.1
+> accept: */*
+> content-type: image/jpeg
+> content-length: 439490
+> 
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+* We are completely uploaded and fine
+< HTTP/2 403 
+< access-control-allow-origin: *
+< alt-svc: h3=":443"; ma=2592000
+< content-type: application/xml
+< date: Fri, 04 Sep 2026 15:23:13 GMT
+< via: 1.1 Caddy
+< content-length: 190
+< 
+* Connection #0 to host s3-api.yiem.cc left intact
+<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message>Forbidden: Invalid signature</Message><Resource>/sommpanion/photo.jpg</Resource><Region>garage</Region></Error>
+
+2026-09-04T15:23:13.796037Z  INFO garage_api_common::generic_server: 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02
+2026-09-04T15:23:13.796077Z DEBUG garage_api_common::generic_server: Request { method: PUT, uri: /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02, version: HTTP/1.1, headers: {"host": "s3-api.yiem.cc", "user-agent": "curl/7.88.1", "content-length": "439490", "accept": "*/*", "content-type": "image/jpeg", "via": "2.0 Caddy", "x-forwarded-for": "110.77.160.154", "x-forwarded-host": "s3-api.yiem.cc", "x-forwarded-proto": "https", "accept-encoding": "gzip"}, body: Body(Streaming) }
+2026-09-04T15:23:13.796181Z DEBUG garage_api_common::generic_server: Endpoint: PutObject
+2026-09-04T15:23:13.796461Z  INFO garage_api_common::generic_server: error 403 Forbidden, Forbidden: Invalid signature in response to 110.77.160.154 (via 192.168.88.21:57926) PUT /sommpanion/photo.jpg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260904%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260904T152217Z&X-Amz-Expires=36000&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=7eb874abb813427a71beab956514c3a0539cacf765013aa194be8517dd081f02
+
+
+
+
+
+
+
+
+
+
 
 
-julia -> DataFrames -> dataframe -> csv
 
-                  dict -> dataframe -> csv

test/runtests.jl

@@ -1,39 +1,693 @@
 using Test
-using GeneralUtils: detect_keyword
+using GeneralUtils
+using GeneralUtils.garageS3
+using HTTP
+
+@testset "garageS3.jl" begin
+
+    # --- SimpleCredentials ---
+
+    @testset "SimpleCredentials construction" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken")
+        @test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE"
+        @test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+        @test creds.token == "sessiontoken"
+    end
+
+    @testset "SimpleCredentials AWS extensions" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1")
+        @test GeneralUtils.garageS3.AWS.credentials(creds) === creds
+        @test GeneralUtils.garageS3.AWS.check_credentials(creds) === creds
+        @test GeneralUtils.garageS3.AWS.refresh!(creds; force=false) === creds
+        @test GeneralUtils.garageS3.AWS.refresh!(creds) === creds
+    end
+
+    # --- GarageConfig ---
+
+    @testset "GarageConfig construction" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds)
+        @test config.endpoint == "https://s3-api.example.com"
+        @test config.region == "us-east-1"
+        @test config.credentials === creds
+        @test config isa GeneralUtils.garageS3.AWS.AbstractAWSConfig
+    end
+
+    @testset "GarageConfig AWS extensions" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
+        @test GeneralUtils.garageS3.AWS.region(config) == "garage"
+        @test GeneralUtils.garageS3.AWS.credentials(config) === creds
+    end
+
+    # --- GeneralUtils.garageS3.AWS.generate_service_url ---
+
+    @testset "generate_service_url with resource starting with /" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
+        url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/mybucket/key")
+        @test url == "https://s3-api.example.com/mybucket/key"
+    end
+
+    @testset "generate_service_url with resource NOT starting with /" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
+        url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "mybucket/key")
+        @test url == "https://s3-api.example.com/mybucket/key"
+    end
+
+    @testset "generate_service_url strips trailing slashes from endpoint" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds)
+        url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
+        @test url == "https://s3-api.example.com/bucket/key"
+    end
+
+    @testset "generate_service_url with multiple trailing slashes" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds)
+        url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
+        @test url == "https://s3-api.example.com/bucket/key"
+    end
+
+    @testset "generate_service_url preserves http endpoint" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds)
+        url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
+        @test url == "http://localhost:3900/bucket/key"
+    end
+
+    @testset "generate_service_url with http endpoint and trailing slash" begin
+        creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
+        config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds)
+        url = GeneralUtils.garageS3.AWS.generate_service_url(config, "s3", "/bucket/key")
+        @test url == "http://localhost:3900/bucket/key"
+    end
+
+    # --- GarageStorage ---
+
+    @testset "GarageStorage default region" begin
+        storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket")
+        @test storage.bucket == "mybucket"
+        @test storage.config.region == "garage"
+        @test storage.config.endpoint == "https://s3-api.example.com"
+    end
+
+    @testset "GarageStorage custom region" begin
+        storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region")
+        @test storage.config.region == "custom-region"
+    end
+
+    @testset "GarageStorage endpoint without https" begin
+        storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket")
+        @test storage.config.endpoint == "http://localhost:3900"
+    end
+
+    @testset "GarageStorage credentials are SimpleCredentials" begin
+        storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket")
+        @test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials
+        @test storage.config.credentials.access_key_id == "mykey"
+        @test storage.config.credentials.secret_key == "mysecret"
+        @test storage.config.credentials.token == ""
+    end
+
+    @testset "GarageStorage returns correct struct type" begin
+        storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
+        @test storage isa GarageStorage
+    end
+
+    # --- s3PutFile callable struct ---
+
+    @testset "s3PutFile callable struct type is correct" begin
+        storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
+        uploader = s3PutFile(storage)
+        @test uploader isa s3PutFile
+    end
+
+    @testset "s3PutFile callable struct construction stores storage" begin
+        storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
+        uploader = s3PutFile(storage)
+        @test uploader.storage === storage
+    end
+
+    # --- Integration tests with real Garage server ---
+
+    @testset "integration: full CRUD lifecycle" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}"
+
+        # Test s3PutFile(storage, key, data) - successful upload
+        result = s3PutFile(storage, test_key, test_data)
+        @test result !== nothing
+        @test haskey(result, :api)
+        @test haskey(result, :web)
+        @test occursin("testbucket", result.api)
+        @test occursin("s3-api", result.api)
+        @test occursin("s3-web", result.web)
+
+        # Test s3GetFile - retrieve the uploaded data
+        downloaded = s3GetFile(storage, test_key)
+        @test downloaded !== nothing
+        @test downloaded isa Vector{UInt8}
+        @test String(downloaded) == test_data
+
+        # Test s3ListFiles - verify key appears in listing
+        keys = s3ListFiles(storage)
+        @test test_key in keys
+
+        # Test s3PutFile callable struct
+        test_key2 = "testrunner-callable-$(GeneralUtils.uuid4snakecase()).json"
+        callable_result = s3PutFile(storage)(test_key2, "callable-data")
+        @test callable_result !== nothing
+        @test callable_result.api != ""
+        @test callable_result.web != ""
+
+        # Verify callable upload
+        downloaded2 = s3GetFile(storage, test_key2)
+        @test String(downloaded2) == "callable-data"
+
+        # Test s3DeleteFile
+        s3DeleteFile(storage, test_key)
+        deleted_data = s3GetFile(storage, test_key)
+        @test deleted_data === nothing
+
+        # Cleanup callable test object
+        s3DeleteFile(storage, test_key2)
+        deleted_data2 = s3GetFile(storage, test_key2)
+        @test deleted_data2 === nothing
+    end
+
+    @testset "integration: nested key CRUD - single level" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "single/slash/test-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"level\": 1, \"nested\": true}"
+
+        # Upload nested key
+        result = s3PutFile(storage, test_key, test_data)
+        @test result isa NamedTuple
+        @test haskey(result, :api)
+        @test haskey(result, :web)
+        @test occursin("single/slash/", result.api)
+        @test occursin("single/slash/", result.web)
+
+        # Download and verify
+        downloaded = s3GetFile(storage, test_key)
+        @test downloaded !== nothing
+        @test String(downloaded) == test_data
+
+        # Verify in listing
+        keys = s3ListFiles(storage)
+        @test test_key in keys
+
+        # Delete and verify gone
+        s3DeleteFile(storage, test_key)
+        @test s3GetFile(storage, test_key) === nothing
+    end
+
+    @testset "integration: nested key CRUD - multi level" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "deep/nested/folder/subfolder/test-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"deeply\": \"nested\", \"path\": \"deep/nested/folder/subfolder/\"}"
+
+        # Upload deeply nested key
+        result = s3PutFile(storage, test_key, test_data)
+        @test result isa NamedTuple
+        @test occursin("deep/nested/folder/subfolder/", result.api)
+        @test occursin("s3-api", result.api)
+        @test occursin("s3-web", result.web)
+
+        # Download and verify
+        downloaded = s3GetFile(storage, test_key)
+        @test downloaded !== nothing
+        @test String(downloaded) == test_data
+
+        # Verify in listing
+        keys = s3ListFiles(storage)
+        @test test_key in keys
+
+        # Delete and verify gone
+        s3DeleteFile(storage, test_key)
+        @test s3GetFile(storage, test_key) === nothing
+    end
+
+    @testset "integration: nested key with callable struct" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "callable/nested/test-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"callable\": true, \"nested\": true}"
+
+        uploader = s3PutFile(storage)
+        result = uploader(test_key, test_data)
+        @test result isa NamedTuple
+        @test occursin("callable/nested/", result.api)
+
+        downloaded = s3GetFile(storage, test_key)
+        @test String(downloaded) == test_data
+
+        s3DeleteFile(storage, test_key)
+        @test s3GetFile(storage, test_key) === nothing
+    end
+
+    @testset "integration: s3GetFile returns nothing for missing key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        result = s3GetFile(storage, "nonexistent-key-$(GeneralUtils.uuid4snakecase()).json")
+        @test result === nothing
+    end
+
+    @testset "integration: binary data upload and download" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-binary-$(GeneralUtils.uuid4snakecase()).bin"
+        binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f]
+
+        result = s3PutFile(storage, test_key, binary_data)
+        @test result !== nothing
+
+        downloaded = s3GetFile(storage, test_key)
+        @test downloaded == binary_data
+
+        s3DeleteFile(storage, test_key)
+        @test s3GetFile(storage, test_key) === nothing
+    end
+
+    @testset "integration: s3ListFiles returns vector of strings" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-list-$(GeneralUtils.uuid4snakecase()).txt"
+        s3PutFile(storage, test_key, "list-test-data")
+
+        keys = s3ListFiles(storage)
+        @test keys isa Vector{String}
+        @test test_key in keys
+
+        s3DeleteFile(storage, test_key)
+    end
+
+    @testset "integration: url format validation" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-url-$(GeneralUtils.uuid4snakecase()).json"
+        result = s3PutFile(storage, test_key, "url-test")
+
+        @test startswith(result.api, "https://s3-api.yiem.cc/testbucket/")
+        @test occursin("/$test_key", result.api)
+        @test startswith(result.web, "https://s3-web.yiem.cc/testbucket/")
+        @test occursin("/$test_key", result.web)
+
+        s3DeleteFile(storage, test_key)
+    end
+
+    # --- Lifecycle configuration tests ---
+
+    @testset "LifecycleExpiration construction with default ID" begin
+        rule = LifecycleExpiration("uploads/", 30)
+        @test rule.prefix == "uploads/"
+        @test rule.days == 30
+        @test rule.enabled == true
+        @test rule.id == "expire-uploads_-30d"
+    end
+
+    @testset "LifecycleExpiration construction with custom ID" begin
+        rule = LifecycleExpiration("data/", 7; id="my-rule")
+        @test rule.prefix == "data/"
+        @test rule.days == 7
+        @test rule.enabled == true
+        @test rule.id == "my-rule"
+    end
+
+    @testset "LifecycleExpiration with disabled flag" begin
+        rule = LifecycleExpiration("temp/", 1; enabled=false)
+        @test rule.enabled == false
+    end
+
+    @testset "LifecycleExpiration with nested prefix" begin
+        rule = LifecycleExpiration("deep/nested/", 90)
+        @test rule.prefix == "deep/nested/"
+    end
+
+    @testset "build_lifecycle_xml generates valid XML" begin
+        rule = LifecycleExpiration("test/", 14; id="test-rule")
+        xml = build_lifecycle_xml(rule)
+        @test occursin("<?xml version=\"1.0\"", xml)
+        @test occursin("<LifecycleConfiguration>", xml)
+        @test occursin("<ID>test-rule</ID>", xml)
+        @test occursin("<Prefix>test/</Prefix>", xml)
+        @test occursin("<Status>Enabled</Status>", xml)
+        @test occursin("<Days>14</Days>", xml)
+        @test occursin("</LifecycleConfiguration>", xml)
+    end
+
+    @testset "build_lifecycle_xml generates disabled rule XML" begin
+        rule = LifecycleExpiration("temp/", 1; enabled=false)
+        xml = build_lifecycle_xml(rule)
+        @test occursin("<Status>Disabled</Status>", xml)
+    end
+
+    @testset "integration: lifecycle CRUD - set and get" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        # Should return nothing when no lifecycle is set
+        lc = s3GetLifecycle(storage)
+        @test lc === nothing
+
+        # Set lifecycle
+        rule = LifecycleExpiration("lifecycle-test/", 45)
+        s3SetLifecycleExpiration(storage, rule)
+
+        # Should return configuration after setting
+        lc = s3GetLifecycle(storage)
+        @test lc !== nothing
+        @test haskey(lc, "Rule")
+
+        # Delete lifecycle
+        s3DeleteLifecycle(storage)
+
+        # Should return nothing after deletion
+        lc = s3GetLifecycle(storage)
+        @test lc === nothing
+    end
+
+    @testset "integration: lifecycle with nested prefix" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        rule = LifecycleExpiration("a/b/c/", 20; id="nested-rule")
+        s3SetLifecycleExpiration(storage, rule)
+
+        lc = s3GetLifecycle(storage)
+        @test lc !== nothing
+        @test haskey(lc, "Rule")
+
+        s3DeleteLifecycle(storage)
+        @test s3GetLifecycle(storage) === nothing
+    end
+
+    @testset "integration: lifecycle enabled/disabled toggle" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        # Set enabled rule
+        rule1 = LifecycleExpiration("toggle/", 10; enabled=true)
+        s3SetLifecycleExpiration(storage, rule1)
+        lc = s3GetLifecycle(storage)
+        @test lc !== nothing
+
+        # Override with disabled rule
+        rule2 = LifecycleExpiration("toggle/", 10; enabled=false)
+        s3SetLifecycleExpiration(storage, rule2)
+        lc = s3GetLifecycle(storage)
+        @test lc !== nothing
+
+        s3DeleteLifecycle(storage)
+    end
+
+    # --- Pre-signed URL tests ---
+
+    @testset "s3SignUrl returns a string" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3SignUrl(storage, "test.json", 3600)
+        @test url isa String
+        @test !isempty(url)
+    end
+
+    @testset "s3SignUrl URL format contains correct endpoint, bucket, key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3SignUrl(storage, "mykey.json", 1800)
+        @test occursin("https://s3-api.yiem.cc", url)
+        @test occursin("/testbucket/mykey.json?", url)
+    end
+
+    @testset "s3SignUrl contains required query parameters" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3SignUrl(storage, "test.json", 3600)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url)
+        @test occursin("X-Amz-Expires=3600", url)
+        @test occursin("X-Amz-Credential=", url)
+        @test occursin("X-Amz-Date=", url)
+        @test occursin("X-Amz-SignedHeaders=", url)
+        @test occursin("X-Amz-Signature=", url)
+    end
+
+    @testset "s3SignUrl with custom expiry duration" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3SignUrl(storage, "test.json", 600)
+        @test occursin("X-Amz-Expires=600", url)
+    end
+
+    @testset "s3SignUrl with GET verb produces different signature than default" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url_get = s3SignUrl(storage, "test.json", 3600; verb="GET")
+        url_default = s3SignUrl(storage, "test.json", 3600)
+        @test occursin("X-Amz-Signature=", url_get)
+        @test occursin("X-Amz-Signature=", url_default)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", url_get)
+    end
+
+    @testset "s3SignUrl with PUT verb and content_type" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url_put = s3SignUrl(storage, "upload.json", 300; verb="PUT", content_type="application/json")
+        url_get = s3SignUrl(storage, "upload.json", 300; verb="GET")
+        @test occursin("X-Amz-Signature=", url_put)
+        @test occursin("X-Amz-Signature=", url_get)
+    end
+
+    @testset "s3SignUrl with special characters in key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        test_key = "folder/sub folder/file with spaces.json"
+        url = s3SignUrl(storage, test_key, 3600)
+        @test occursin("testbucket/", url)
+    end
+
+    @testset "s3SignUrl with deeply nested key" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        test_key = "a/b/c/d/e/deep-file.json"
+        url = s3SignUrl(storage, test_key, 3600)
+        @test occursin("/testbucket/a/b/c/d/e/deep-file.json", url)
+    end
+
+    @testset "integration: pre-signed URL can download existing file" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-presign-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"presign\": true, \"test\": \"$(GeneralUtils.uuid4snakecase())\"}"
+        s3PutFile(storage, test_key, test_data)
+
+        url = s3SignUrl(storage, test_key, 300)
+        @test occursin("https://s3-api.yiem.cc/testbucket/", url)
+
+        s3DeleteFile(storage, test_key)
+    end
+
+    # --- s3SignUrl end-to-end: PUT upload, GET download, DELETE ---
+
+    @testset "integration: s3SignUrl PUT upload via pre-signed URL" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-signurl-put-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"signed\": true, \"upload\": \"put\"}"
+        content_type = "application/json"
+
+        # Generate pre-signed URL for PUT
+        put_url = s3SignUrl(storage, test_key, 300; verb="PUT", content_type=content_type)
+        @test occursin("X-Amz-Signature=", put_url)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", put_url)
+        @test occursin("X-Amz-SignedHeaders=", put_url)
+
+        # Upload via HTTP request using the signed URL
+        response = HTTP.request(
+            "PUT",
+            put_url,
+            ["Content-Type" => content_type],
+            test_data
+        )
+        @test response.status == 200
+
+        # Verify upload via s3GetFile
+        downloaded = s3GetFile(storage, test_key)
+        @test downloaded !== nothing
+        @test String(downloaded) == test_data
+
+        # Cleanup
+        s3DeleteFile(storage, test_key)
+    end
+
+    @testset "integration: s3SignUrl GET download via pre-signed URL" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-signurl-get-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"signed\": true, \"download\": \"get\"}"
+
+        # Upload via s3PutFile first
+        s3PutFile(storage, test_key, test_data)
+
+        # Generate pre-signed URL for GET
+        get_url = s3SignUrl(storage, test_key, 300; verb="GET")
+        @test occursin("X-Amz-Signature=", get_url)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", get_url)
+
+        # Download via HTTP GET request using the signed URL
+        response = HTTP.request("GET", get_url)
+        @test response.status == 200
+        @test String(response.body) == test_data
+
+        # Cleanup
+        s3DeleteFile(storage, test_key)
+    end
+
+    @testset "integration: s3SignUrl DELETE via pre-signed URL" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+
+        test_key = "testrunner-signurl-delete-$(GeneralUtils.uuid4snakecase()).json"
+        test_data = "{\"signed\": true, \"delete\": true}"
+
+        # Upload first
+        s3PutFile(storage, test_key, test_data)
+
+        # Verify it exists
+        @test s3GetFile(storage, test_key) !== nothing
+
+        # Generate pre-signed URL for DELETE
+        delete_url = s3SignUrl(storage, test_key, 300; verb="DELETE")
+        @test occursin("X-Amz-Signature=", delete_url)
+        @test occursin("X-Amz-Algorithm=AWS4-HMAC-SHA256", delete_url)
+
+        # Delete via HTTP DELETE request using the signed URL
+        response = HTTP.request("DELETE", delete_url)
+        @test response.status == 204
+
+        # Verify deletion
+        @test s3GetFile(storage, test_key) === nothing
+    end
+
+    @testset "integration: s3SignUrlWeb generates web endpoint URL" begin
+        storage = GarageStorage(
+            "https://s3-api.yiem.cc",
+            "GKb080154a2e5b19100b1b2c6e",
+            "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
+            "testbucket"
+        )
+        url = s3SignUrlWeb(storage, "web-test.json", 3600)
+        @test url isa String
+        @test occursin("https://s3-web.yiem.cc", url)
+        @test occursin("/testbucket/web-test.json?", url)
+        @test occursin("X-Amz-Signature=", url)
+    end
 
-@testset "detect_keyword tests" begin
-    @test detect_keyword(["test"], "this is a test") == Dict("test" => 1)
-    
-    @test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1)
-    
-    @test detect_keyword(["cat"], "category") == Dict("cat" => 1)
-    
-    @test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0)
-    
-    @test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0)
-    
-    @test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1)
-    
-    @test detect_keyword(["word"], "") == Dict("word" => 0)
-    
-    @test detect_keyword(String[], "some text") == Dict{String, Integer}()
-    
-    @test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1)
-    
-    multiline_text = """
-    first line
-    second line
-    first word
-    """
-    @test detect_keyword(["first"], multiline_text) == Dict("first" => 2)
-    
-    @test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1)
-    
-    @test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1)
 end
-
-
-
-
-
-