update

Reviewed-on: #7

AI_prompt.txt

@@ -12,3 +12,42 @@ Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement
 
 
 
+Create a walkthrough for Julia service-A service sending a mix-content chat message to Julia service-B. the chat message must includes 
+
+
+
+
+
+
+I updated the following:
+- NATSBridge.jl. Essentially I add NATS_connection keyword and new publish_message function to support the keyword. 
+
+Use them and ONLY them as ground truth.
+
+Then update the following files accordingly:
+- architecture.md
+- implementation.md
+
+All API should be semantically consistent and naming should be consistent across the board.
+
+
+
+
+
+
+
+
+
+Task: Update NATSBridge.js to reflect recent changes in NATSBridge.jl and docs
+
+Context: NATSBridge.jl and docs has been updated.
+
+Requirements:
+
+Source of Truth: Treat the updated NATSBridge.jl and docs as the definitive source.
+API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages.
+Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency.
+
+
+
+

Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.12.4"
+julia_version = "1.12.5"
 manifest_format = "2.0"
-project_hash = "be1e3c2d8b7f4f0ee7375c94aaf704ce73ba57b9"
+project_hash = "b632f853bcf5355f5c53ad3efa7a19f70444dc6c"
 
 [[deps.AliasTables]]
 deps = ["PtrArrays", "Random"]
@@ -436,6 +436,12 @@ git-tree-sha1 = "d9d9a189fb9155a460e6b5e8966bf6a66737abf8"
 uuid = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
 version = "0.1.0"
 
+[[deps.NATSBridge]]
+deps = ["Arrow", "DataFrames", "Dates", "GeneralUtils", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"]
+path = "."
+uuid = "f2724d33-f338-4a57-b9f8-1be882570d10"
+version = "0.4.1"
+
 [[deps.NanoDates]]
 deps = ["Dates", "Parsers"]
 git-tree-sha1 = "850a0557ae5934f6e67ac0dc5ca13d0328422d1f"

Plik_API_document.md

@@ -1,194 +0,0 @@
-### API
-Plik server expose a REST-full API to manage uploads and get files :
-
-Get and create upload :
- 
-   - **POST**        /upload
-     - Params (json object in request body) :
-      - oneshot (bool)
-      - stream (bool)
-      - removable (bool)
-      - ttl (int)
-      - login (string)
-      - password (string)
-      - files (see below)
-     - Return :
-         JSON formatted upload object.
-         Important fields :
-           - id (required to upload files)
-           - uploadToken (required to upload/remove files)
-           - files (see below)
-
-   For stream mode you need to know the file id before the upload starts as it will block.
-   File size and/or file type also need to be known before the upload starts as they have to be printed 
-   in HTTP response headers.
-   To get the file ids pass a "files" json object with each file you are about to upload.
-   Fill the reference field with an arbitrary string to avoid matching file ids using the fileName field.
-   This is also used to notify of MISSING files when file upload is not yet finished or has failed.
-  ```
-  "files" : [
-    {
-      "fileName": "file.txt",
-      "fileSize": 12345,
-      "fileType": "text/plain",
-      "reference": "0"
-    },...
-  ]
-  ```
-  
-   - **GET** /upload/:uploadid:
-     - Get upload metadata (files list, upload date, ttl,...)
-
-Upload file :
-
-   - **POST** /$mode/:uploadid:/:fileid:/:filename:
-     - Request body must be a multipart request with a part named "file" containing file data.
-
-   - **POST** /file/:uploadid:
-     - Same as above without passing file id, won't work for stream mode.
-     
-   - **POST** /:
-     - Quick mode, automatically create an upload with default parameters and add the file to it.
-
-Get file :
-
-  - **HEAD** /$mode/:uploadid:/:fileid:/:filename:
-    - Returns only HTTP headers. Useful to know Content-Type and Content-Length without downloading the file. Especially if upload has OneShot option enabled.
-
-  - **GET**  /$mode/:uploadid:/:fileid:/:filename:
-    - Download file. Filename **MUST** match. A browser, might try to display the file if it's a jpeg for example. You may try to force download with ?dl=1 in url.
-
-  - **GET**  /archive/:uploadid:/:filename:
-    - Download uploaded files in a zip archive. :filename: must end with .zip
-
-Remove file :
-
-   - **DELETE** /$mode/:uploadid:/:fileid:/:filename:
-     - Delete file. Upload **MUST** have "removable" option enabled.
-
-Show server details :
-
-   - **GET** /version
-     - Show plik server version, and some build information (build host, date, git revision,...)
-
-   - **GET** /config
-     - Show plik server configuration (ttl values, max file size, ...)
-
-   - **GET** /stats
-     - Get server statistics ( upload/file count, user count, total size used )
-     - Admin only
-
-User authentication :
-
-   - 
-   Plik can authenticate users using Google and/or OVH third-party API.   
-   The /auth API is designed for the Plik web application nevertheless if you want to automatize it be sure to provide a valid
-   Referrer HTTP header and forward all session cookies.   
-   Plik session cookies have the "secure" flag set, so they can only be transmitted over secure HTTPS connections.   
-   To avoid CSRF attacks the value of the plik-xsrf cookie MUST be copied in the X-XSRFToken HTTP header of each
-   authenticated request.   
-   Once authenticated a user can generate upload tokens. Those tokens can be used in the X-PlikToken HTTP header used to link
-   an upload to the user account. It can be put in the ~/.plikrc file of the Plik command line client.   
-   
-   - **Local** :
-      - You'll need to create users using the server command line
-   
-   - **Google** :
-      - You'll need to create a new application in the [Google Developper Console](https://console.developers.google.com)
-      - You'll be handed a Google API ClientID and a Google API ClientSecret that you'll need to put in the plikd.cfg file
-      - Do not forget to whitelist valid origin and redirect url ( https://yourdomain/auth/google/callback ) for your domain
-   
-   - **OVH** :
-      - You'll need to create a new application in the OVH API : https://eu.api.ovh.com/createApp/
-      - You'll be handed an OVH application key and an OVH application secret key that you'll need to put in the plikd.cfg file
-
-   - **GET** /auth/google/login
-      - Get Google user consent URL. User have to visit this URL to authenticate
-
-   - **GET** /auth/google/callback
-     - Callback of the user consent dialog
-     - The user will be redirected back to the web application with a Plik session cookie at the end of this call
-
-   - **GET** /auth/ovh/login
-     - Get OVH user consent URL. User have to visit this URL to authenticate
-     - The response will contain a temporary session cookie to forward the API endpoint and OVH consumer key to the callback
-
-   - **GET** /auth/ovh/callback
-     - Callback of the user consent dialog. 
-     - The user will be redirected back to the web application with a Plik session cookie at the end of this call
-
-   - **POST** /auth/local/login
-     - Params :
-       - login : user login
-       - password : user password
-
-   - **GET** /auth/logout
-     - Invalidate Plik session cookies
-
-   - **GET** /me
-     - Return basic user info ( ID, name, email ) and tokens
-
-   - **DELETE** /me
-     - Remove user account.
-
-   - **GET** /me/token
-     - List user tokens
-      - This call use pagination
-
-   - **POST** /me/token
-     - Create a new upload token
-     - A comment can be passed in the json body
-
-   - **DELETE** /me/token/{token}
-     - Revoke an upload token
-
-   - **GET** /me/uploads
-     - List user uploads
-     - Params :
-        - token : filter by token
-      - This call use pagination
-
-   - **DELETE** /me/uploads
-     - Remove all uploads linked to a user account
-     - Params :
-        - token : filter by token
-
-   - **GET** /me/stats
-     - Get user statistics ( upload/file count, total size used )
-
-   - **GET** /users
-     - List all users
-     - This call use pagination
-     - Admin only 
-
-QRCode :
-
-   - **GET** /qrcode
-     - Generate a QRCode image from an url
-     - Params :
-        - url  : The url you want to store in the QRCode
-        - size : The size of the generated image in pixels (default: 250, max: 1000)
-
-
-$mode can be "file" or "stream" depending if stream mode is enabled. See FAQ for more details.
-
-Examples :
-```sh
-Create an upload (in the json response, you'll have upload id and upload token)
-$ curl -X POST http://127.0.0.1:8080/upload
-
-Create a OneShot upload
-$ curl -X POST -d '{ "OneShot" : true }' http://127.0.0.1:8080/upload
-
-Upload a file to upload
-$ curl -X POST --header "X-UploadToken: M9PJftiApG1Kqr81gN3Fq1HJItPENMhl" -F "file=@test.txt" http://127.0.0.1:8080/file/IsrIPIsDskFpN12E
-
-Get headers
-$ curl -I http://127.0.0.1:8080/file/IsrIPIsDskFpN12E/sFjIeokH23M35tN4/test.txt
-HTTP/1.1 200 OK
-Content-Disposition: filename=test.txt
-Content-Length: 3486
-Content-Type: text/plain; charset=utf-8
-Date: Fri, 15 May 2015 09:16:20 GMT
-
-```

Project.toml

@@ -1,8 +1,21 @@
+name = "NATSBridge"
+uuid = "f2724d33-f338-4a57-b9f8-1be882570d10"
+version = "0.4.3"
+authors = ["narawat <narawat@gmail.com>"]
+
 [deps]
 Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
+PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[compat]
+Base64 = "1.11.0"
+JSON = "1.4.0"

README.md

@@ -0,0 +1,495 @@
+# NATSBridge
+
+A high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![NATS](https://img.shields.io/badge/NATS-Enabled-green.svg)](https://nats.io)
+
+---
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Features](#features)
+- [Architecture](#architecture)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+- [API Reference](#api-reference)
+- [Payload Types](#payload-types)
+- [Transport Strategies](#transport-strategies)
+- [Examples](#examples)
+- [Testing](#testing)
+- [License](#license)
+
+---
+
+## Overview
+
+NATSBridge enables seamless communication for Julia applications through NATS, with intelligent transport selection based on payload size:
+
+| Transport | Payload Size | Method |
+|-----------|--------------|--------|
+| **Direct** | < 1MB | Sent directly via NATS (Base64 encoded) |
+| **Link** | >= 1MB | Uploaded to HTTP file server, URL sent via NATS |
+
+### Use Cases
+
+- **Chat Applications**: Text, images, audio, video in a single message
+- **File Transfer**: Efficient transfer of large files using claim-check pattern
+- **Streaming Data**: Sensor data, telemetry, and analytics pipelines
+
+
+---
+
+## Features
+
+- ✅ **Bi-directional messaging** for Julia applications
+- ✅ **Multi-payload support** - send multiple payloads with different types in one message
+- ✅ **Automatic transport selection** - direct vs link based on payload size
+- ✅ **Claim-Check pattern** for payloads > 1MB
+- ✅ **Apache Arrow IPC** support for tabular data (zero-copy reading)
+- ✅ **Exponential backoff** for reliable file server downloads
+- ✅ **Correlation ID tracking** for message tracing
+- ✅ **Reply-to support** for request-response patterns
+- ✅ **JetStream support** for message replay and durability
+
+
+---
+
+## Architecture
+
+### System Components
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                     NATSBridge Architecture                         │
+├─────────────────────────────────────────────────────────────────────┤
+│  ┌──────────────┐                    │                              │
+│  │   Julia      │                    ▼                              │
+│  │  (NATS.jl)   │    ┌─────────────────────────┐                    │
+│  └──────────────┘    │    NATS                 │                    │
+│                      │    (Message Broker)     │                    │
+│                      └─────────────────────────┘                    │
+│                              │                                      │
+│                              ▼                                      │
+│                    ┌──────────────────────┐                         │
+│                    │  File Server         │                         │
+│                    │  (HTTP Upload/Get)   │                         │
+│                    └──────────────────────┘                         │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+### Message Flow
+
+1. **Sender** creates a message envelope with payloads
+2. **NATSBridge** serializes and encodes payloads based on type
+3. **Transport Decision**: Small payloads go directly to NATS, large payloads are uploaded to file server
+4. **NATS** routes messages to subscribers
+5. **Receiver** fetches payloads (from NATS or file server)
+6. **NATSBridge** deserializes and decodes payloads
+
+---
+
+## Installation
+
+### Prerequisites
+
+- **NATS Server** (v2.10+ recommended)
+- **HTTP File Server** (optional, for payloads > 1MB)
+
+### Julia
+
+```julia
+using Pkg
+Pkg.add("NATS")
+Pkg.add("https://git.yiem.cc/ton/NATSBridge")
+```
+
+---
+
+## Quick Start
+
+### Step 1: Start NATS Server
+
+```bash
+docker run -p 4222:4222 nats:latest
+```
+
+### Step 2: Start HTTP File Server (Optional)
+
+```bash
+# Create a directory for file uploads
+mkdir -p /tmp/fileserver
+
+# Start HTTP file server
+python3 -m http.server 8080 --directory /tmp/fileserver
+```
+
+### Step 3: Send Your First Message
+
+#### Julia
+
+```julia
+using NATSBridge
+
+# Send a text message
+data = [("message", "Hello World", "text")]
+env, env_json_str = NATSBridge.smartsend("/chat/room1", data; broker_url="nats://localhost:4222")
+println("Message sent!")
+```
+
+### Step 4: Receive Messages
+
+#### Julia
+
+```julia
+using NATS, NATSBridge
+
+# Configuration
+const SUBJECT = "/chat/room1"
+const NATS_URL = "nats://localhost:4222"
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+# Receiver: Listen for messages - msg comes from the callback
+function test_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Receive and process message
+        env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler)
+        for (dataname, data, type) in env["payloads"]
+            println("Received $dataname: $data")
+        end
+    end
+    
+    # Keep listening for 120 seconds
+    sleep(120)
+    NATS.drain(conn)
+end
+
+test_receive()
+```
+
+---
+
+## API Reference
+
+### smartsend
+
+Sends data either directly via NATS or via a fileserver URL, depending on payload size.
+
+#### Julia
+
+```julia
+using NATSBridge
+
+env, env_json_str = NATSBridge.smartsend(
+    subject,                           # NATS subject
+    data::AbstractArray{Tuple{String, Any, String}};  # List of (dataname, data, type)
+    broker_url::String = "nats://localhost:4222",
+    fileserver_url = "http://localhost:8080",
+    fileserver_upload_handler::Function = plik_oneshot_upload,
+    size_threshold::Int = 1_000_000,
+    correlation_id::Union{String, Nothing} = nothing,
+    msg_purpose::String = "chat",
+    sender_name::String = "NATSBridge",
+    receiver_name::String = "",
+    receiver_id::String = "",
+    reply_to::String = "",
+    reply_to_msg_id::String = "",
+    is_publish::Bool = true,          # Whether to automatically publish to NATS
+    NATS_connection::Union{NATS.Connection, Nothing} = nothing  # Pre-existing NATS connection (optional, saves connection overhead)
+)
+# Returns: (msgEnvelope_v1, JSON string)
+# - env: msgEnvelope_v1 object with all envelope metadata and payloads
+# - env_json_str: JSON string representation of the envelope for publishing
+```
+
+### smartreceive
+
+Receives and processes messages from NATS, handling both direct and link transport.
+
+#### Julia
+
+```julia
+using NATSBridge
+
+# Note: msg is a NATS.Msg object passed from the subscription callback
+env = NATSBridge.smartreceive(
+    msg::NATS.Msg;
+    fileserver_download_handler::Function = _fetch_with_backoff,
+    max_retries::Int = 5,
+    base_delay::Int = 100,
+    max_delay::Int = 5000
+)
+# Returns: Dict with envelope metadata and payloads array
+```
+
+### publish_message
+
+Publish a message to a NATS subject. This function is available in Julia with two overloads:
+
+#### Julia
+
+**Using broker URL (creates new connection):**
+```julia
+using NATSBridge, NATS
+
+# Publish with URL - creates a new connection
+NATSBridge.publish_message(
+    "nats://localhost:4222",  # broker_url
+    "/chat/room1",            # subject
+    "{\"correlation_id\":\"abc123\"}",  # message
+    "abc123"                  # correlation_id
+)
+```
+
+**Using pre-existing connection (saves connection overhead):**
+```julia
+using NATSBridge, NATS
+
+# Create connection once and reuse
+conn = NATS.connect("nats://localhost:4222")
+NATSBridge.publish_message(conn, "/chat/room1", "{\"correlation_id\":\"abc123\"}", "abc123")
+# Connection is automatically drained after publish
+```
+
+---
+
+## Payload Types
+
+| Type | Description | Serialization |
+|------|-------------|---------------|
+| `text` | Plain text strings | UTF-8 bytes |
+| `dictionary` | JSON-serializable dictionaries | JSON |
+| `table` | Tabular data (DataFrames, arrays) | Apache Arrow IPC |
+| `image` | Image data (PNG, JPG) | Raw bytes |
+| `audio` | Audio data (WAV, MP3) | Raw bytes |
+| `video` | Video data (MP4, AVI) | Raw bytes |
+| `binary` | Generic binary data | Raw bytes |
+
+---
+
+## Transport Strategies
+
+### Direct Transport (Payloads < 1MB)
+
+Small payloads are sent directly via NATS with Base64 encoding.
+
+#### Julia
+```julia
+data = [("message", "Hello", "text")]
+smartsend("/topic", data)
+```
+
+### Link Transport (Payloads >= 1MB)
+
+Large payloads are uploaded to an HTTP file server.
+
+#### Julia
+```julia
+data = [("file", large_data, "binary")]
+smartsend("/topic", data; fileserver_url="http://localhost:8080")
+```
+
+---
+
+## Examples
+
+### Example 1: Chat with Mixed Content
+
+Send text, small image, and large file in one message.
+
+#### Julia
+```julia
+using NATSBridge
+
+data = [
+    ("message_text", "Hello!", "text"),
+    ("user_avatar", image_data, "image"),
+    ("large_document", large_file_data, "binary")
+]
+
+env, env_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080")
+```
+
+### Example 2: Dictionary Exchange
+
+Send configuration data between platforms.
+
+#### Julia
+```julia
+using NATSBridge
+
+config = Dict(
+    "wifi_ssid" => "MyNetwork",
+    "wifi_password" => "password123",
+    "update_interval" => 60
+)
+
+data = [("config", config, "dictionary")]
+env, env_json_str = NATSBridge.smartsend("/device/config", data)
+```
+
+### Example 3: Table Data (Arrow IPC)
+
+Send tabular data using Apache Arrow IPC format.
+
+#### Julia
+```julia
+using NATSBridge
+using DataFrames
+
+df = DataFrame(
+    id = [1, 2, 3],
+    name = ["Alice", "Bob", "Charlie"],
+    score = [95, 88, 92]
+)
+
+data = [("students", df, "table")]
+env, env_json_str = NATSBridge.smartsend("/data/analysis", data)
+```
+
+### Example 4: Request-Response Pattern with Envelope JSON
+
+Bi-directional communication with reply-to support. The `smartsend` function now returns both the envelope object and a JSON string that can be published directly.
+
+#### Julia (Requester)
+```julia
+using NATSBridge
+
+env, env_json_str = NATSBridge.smartsend(
+    "/device/command",
+    [("command", Dict("action" => "read_sensor"), "dictionary")];
+    broker_url="nats://localhost:4222",
+    reply_to="/device/response"
+)
+```
+
+#### Julia (Responder)
+```julia
+using NATS, NATSBridge
+
+# Configuration
+const SUBJECT = "/device/command"
+const NATS_URL = "nats://localhost:4222"
+
+function test_responder()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        env = NATSBridge.smartreceive(msg, fileserver_download_handler=_fetch_with_backoff)
+        
+        # Extract reply_to from the envelope metadata
+        reply_to = env["reply_to"]
+        
+        for (dataname, data, type) in env["payloads"]
+            if dataname == "command" && data["action"] == "read_sensor"
+                response = Dict("sensor_id" => "sensor-001", "value" => 42.5)
+                # Send response to the reply_to subject from the request
+                if !isempty(reply_to)
+                    smartsend(reply_to, [("data", response, "dictionary")])
+                end
+            end
+        end
+    end
+    
+    sleep(120)
+    NATS.drain(conn)
+end
+
+test_responder()
+```
+
+### Example 5: IoT Device Sensor Data
+
+IoT device sending sensor data.
+
+#### Julia (Receiver)
+```julia
+using NATS, NATSBridge
+
+# Configuration
+const SUBJECT = "/device/sensors"
+const NATS_URL = "nats://localhost:4222"
+
+function test_receiver()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler)
+        for (dataname, data, type) in env["payloads"]
+            if dataname == "temperature"
+                println("Temperature: $data")
+            elseif dataname == "humidity"
+                println("Humidity: $data")
+            end
+        end
+    end
+    
+    sleep(120)
+    NATS.drain(conn)
+end
+
+test_receiver()
+```
+
+---
+
+## Testing
+
+Run the test scripts to verify functionality:
+
+### Julia
+
+```julia
+# Text message exchange
+julia test/test_julia_text_sender.jl
+julia test/test_julia_text_receiver.jl
+
+# Dictionary exchange
+julia test/test_julia_dict_sender.jl
+julia test/test_julia_dict_receiver.jl
+
+# File transfer
+julia test/test_julia_file_sender.jl
+julia test/test_julia_file_receiver.jl
+
+# Mixed payload types
+julia test/test_julia_mix_payloads_sender.jl
+julia test/test_julia_mix_payloads_receiver.jl
+
+# Table exchange
+julia test/test_julia_table_sender.jl
+julia test/test_julia_table_receiver.jl
+```
+
+---
+
+## License
+
+MIT License
+
+Copyright (c) 2026 NATSBridge Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

architecture.md

@@ -1,294 +0,0 @@
-# Architecture Documentation: Bi-Directional Data Bridge (Julia ↔ JavaScript)
-
-## Overview
-
-This document describes the architecture for a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
-
-## Architecture Diagram
-
-```mermaid
-flowchart TD
-    subgraph Client
-        JS[JavaScript Client]
-        JSApp[Application Logic]
-    end
-
-    subgraph Server
-        Julia[Julia Service]
-        NATS[NATS Server]
-        FileServer[HTTP File Server]
-    end
-
-    JS -->|Control/Small Data| JSApp
-    JSApp -->|NATS| NATS
-    NATS -->|NATS| Julia
-    Julia -->|NATS| NATS
-    Julia -->|HTTP POST| FileServer
-    JS -->|HTTP GET| FileServer
-
-    style JS fill:#e1f5fe
-    style Julia fill:#e8f5e9
-    style NATS fill:#fff3e0
-    style FileServer fill:#f3e5f5
-```
-
-## System Components
-
-### 1. Unified JSON Envelope Schema
-
-All messages use a standardized envelope format:
-
-```json
-{
-  "correlation_id": "uuid-v4-string",
-  "type": "json|table|binary",
-  "transport": "direct|link",
-  "payload": "base64-encoded-string",  // Only if transport=direct
-  "url": "http://fileserver/path/to/data",  // Only if transport=link
-  "metadata": {
-    "content_type": "application/octet-stream",
-    "content_length": 123456,
-    "format": "arrow_ipc_stream"
-  }
-}
-```
-
-### 2. Transport Strategy Decision Logic
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│                     SmartSend Function                        │
-└─────────────────────────────────────────────────────────────┘
-                            │
-                            ▼
-┌─────────────────────────────────────────────────────────────┐
-│  Is payload size < 1MB?                                     │
-└─────────────────────────────────────────────────────────────┘
-                            │
-          ┌─────────────────┴─────────────────┐
-          ▼                                   ▼
-    ┌─────────────────┐                 ┌─────────────────┐
-    │  Direct Path    │                 │  Link Path      │
-    │  (< 1MB)        │                 │  (> 1MB)        │
-    │                 │                 │                 │
-    │ • Serialize to  │                 │ • Serialize to  │
-    │   IOBuffer      │                 │   IOBuffer      │
-    │ • Base64 encode │                 │ • Upload to     │
-    │ • Publish to    │                 │   HTTP Server   │
-    │   NATS          │                 │ • Publish to    │
-    │                 │                 │   NATS with URL │
-    └─────────────────┘                 └─────────────────┘
-```
-
-### 3. Julia Module Architecture
-
-```mermaid
-graph TD
-    subgraph JuliaModule
-        SmartSendJulia[SmartSend Julia]
-        SizeCheck[Size Check]
-        DirectPath[Direct Path]
-        LinkPath[Link Path]
-        HTTPClient[HTTP Client]
-    end
-
-    SmartSendJulia --> SizeCheck
-    SizeCheck -->|< 1MB| DirectPath
-    SizeCheck -->|>= 1MB| LinkPath
-    LinkPath --> HTTPClient
-
-    style JuliaModule fill:#c5e1a5
-```
-
-### 4. JavaScript Module Architecture
-
-```mermaid
-graph TD
-    subgraph JSModule
-        SmartSendJS[SmartSend JS]
-        SmartReceiveJS[SmartReceive JS]
-        JetStreamConsumer[JetStream Pull Consumer]
-        ApacheArrow[Apache Arrow]
-    end
-
-    SmartSendJS --> NATS
-    SmartReceiveJS --> JetStreamConsumer
-    JetStreamConsumer --> ApacheArrow
-
-    style JSModule fill:#f3e5f5
-```
-
-## Implementation Details
-
-### Julia Implementation
-
-#### Dependencies
-- `NATS.jl` - Core NATS functionality
-- `Arrow.jl` - Arrow IPC serialization
-- `JSON3.jl` - JSON parsing
-- `HTTP.jl` - HTTP client for file server
-- `Dates.jl` - Timestamps for logging
-
-#### SmartSend Function
-
-```julia
-function SmartSend(
-    subject::String,
-    data::Any,
-    type::String = "json";
-    nats_url::String = "nats://localhost:4222",
-    fileserver_url::String = "http://localhost:8080/upload",
-    size_threshold::Int = 1_000_000  # 1MB
-)
-```
-
-**Flow:**
-1. Serialize data to Arrow IPC stream (if table)
-2. Check payload size
-3. If < threshold: publish directly to NATS with Base64-encoded payload
-4. If >= threshold: upload to HTTP server, publish NATS with URL
-
-#### SmartReceive Handler
-
-```julia
-function SmartReceive(msg::NATS.Message)
-    # Parse envelope
-    # Check transport type
-    # If direct: decode Base64 payload
-    # If link: fetch from URL with exponential backoff
-    # Deserialize Arrow IPC to DataFrame
-end
-```
-
-### JavaScript Implementation
-
-#### Dependencies
-- `nats.js` - Core NATS functionality
-- `apache-arrow` - Arrow IPC serialization
-- `uuid` - Correlation ID generation
-
-#### SmartSend Function
-
-```javascript
-async function SmartSend(subject, data, type = 'json', options = {})
-```
-
-**Flow:**
-1. Serialize data to Arrow IPC buffer (if table)
-2. Check payload size
-3. If < threshold: publish directly to NATS
-4. If >= threshold: upload to HTTP server, publish NATS with URL
-
-#### SmartReceive Handler
-
-```javascript
-async function SmartReceive(msg, options = {})
-```
-
-**Flow:**
-1. Parse envelope
-2. Check transport type
-3. If direct: decode Base64 payload
-4. If link: fetch with exponential backoff
-5. Deserialize Arrow IPC with zero-copy
-
-## Scenario Implementations
-
-### Scenario 1: Command & Control (Small JSON)
-
-**Julia (Receiver):**
-```julia
-# Subscribe to control subject
-# Parse JSON envelope
-# Execute simulation with parameters
-# Send acknowledgment
-```
-
-**JavaScript (Sender):**
-```javascript
-// Create small JSON config
-// Send via SmartSend with type="json"
-```
-
-### Scenario 2: Deep Dive Analysis (Large Arrow Table)
-
-**Julia (Sender):**
-```julia
-# Create large DataFrame
-# Convert to Arrow IPC stream
-# Check size (> 1MB)
-# Upload to HTTP server
-# Publish NATS with URL
-```
-
-**JavaScript (Receiver):**
-```javascript
-// Receive NATS message with URL
-// Fetch data from HTTP server
-// Parse Arrow IPC with zero-copy
-// Load into Perspective.js or D3
-```
-
-### Scenario 3: Live Audio Processing
-
-**JavaScript (Sender):**
-```javascript
-// Capture audio chunk
-// Send as binary with metadata headers
-// Use SmartSend with type="audio"
-```
-
-**Julia (Receiver):**
-```julia
-// Receive audio data
-// Perform FFT or AI transcription
-// Send results back (JSON + Arrow table)
-```
-
-### Scenario 4: Catch-Up (JetStream)
-
-**Julia (Producer):**
-```julia
-# Publish to JetStream
-# Include metadata for temporal tracking
-```
-
-**JavaScript (Consumer):**
-```javascript
-// Connect to JetStream
-// Request replay from last 10 minutes
-// Process historical and real-time messages
-```
-
-## Performance Considerations
-
-### Zero-Copy Reading
-- Use Arrow's memory-mapped file reading
-- Avoid unnecessary data copying during deserialization
-- Use Apache Arrow's native IPC reader
-
-### Exponential Backoff
-- Implement exponential backoff for HTTP link fetching
-- Maximum retry count: 5
-- Base delay: 100ms, max delay: 5000ms
-
-### Correlation ID Logging
-- Log correlation_id at every stage
-- Include: send, receive, serialize, deserialize
-- Use structured logging format
-
-## Testing Strategy
-
-### Unit Tests
-- Test SmartSend with various payload sizes
-- Test SmartReceive with direct and link transport
-- Test Arrow IPC serialization/deserialization
-
-### Integration Tests
-- Test full flow with NATS server
-- Test large data transfer (> 100MB)
-- Test audio processing pipeline
-
-### Performance Tests
-- Measure throughput for small payloads
-- Measure throughput for large payloads

docs/IMPLEMENTATION.md

@@ -1,321 +0,0 @@
-# Implementation Guide: Bi-Directional Data Bridge
-
-## Overview
-
-This document describes the implementation of the high-performance, bi-directional data bridge between Julia and JavaScript services using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
-
-## Architecture
-
-The implementation follows the Claim-Check pattern:
-
-```
-┌─────────────────────────────────────────────────────────────────────────┐
-│                         SmartSend Function                              │
-└─────────────────────────────────────────────────────────────────────────┘
-                             │
-                             ▼
-┌─────────────────────────────────────────────────────────────────────────┐
-│  Is payload size < 1MB?                                                 │
-└─────────────────────────────────────────────────────────────────────────┘
-                             │
-           ┌─────────────────┴─────────────────┐
-           ▼                                   ▼
-     ┌─────────────────┐                 ┌─────────────────┐
-     │  Direct Path    │                 │  Link Path      │
-     │  (< 1MB)        │                 │  (> 1MB)        │
-     │                 │                 │                 │
-     │ • Serialize to  │                 │ • Serialize to  │
-     │   IOBuffer      │                 │   IOBuffer      │
-     │ • Base64 encode │                 │ • Upload to     │
-     │ • Publish to    │                 │   HTTP Server   │
-     │   NATS          │                 │ • Publish to    │
-     │                 │                 │   NATS with URL │
-     └─────────────────┘                 └─────────────────┘
-```
-
-## Files
-
-### Julia Module: [`src/julia_bridge.jl`](../src/julia_bridge.jl)
-
-The Julia implementation provides:
-
-- **[`MessageEnvelope`](../src/julia_bridge.jl)**: Struct for the unified JSON envelope
-- **[`SmartSend()`](../src/julia_bridge.jl)**: Handles transport selection based on payload size
-- **[`SmartReceive()`](../src/julia_bridge.jl)**: Handles both direct and link transport
-
-### JavaScript Module: [`src/js_bridge.js`](../src/js_bridge.js)
-
-The JavaScript implementation provides:
-
-- **`MessageEnvelope` class**: For the unified JSON envelope
-- **[`SmartSend()`](../src/js_bridge.js)**: Handles transport selection based on payload size
-- **[`SmartReceive()`](../src/js_bridge.js)**: Handles both direct and link transport
-
-## Installation
-
-### Julia Dependencies
-
-```julia
-using Pkg
-Pkg.add("NATS")
-Pkg.add("Arrow")
-Pkg.add("JSON3")
-Pkg.add("HTTP")
-Pkg.add("UUIDs")
-Pkg.add("Dates")
-```
-
-### JavaScript Dependencies
-
-```bash
-npm install nats.js apache-arrow uuid base64-url
-```
-
-## Usage Tutorial
-
-### Step 1: Start NATS Server
-
-```bash
-docker run -p 4222:4222 nats:latest
-```
-
-### Step 2: Start HTTP File Server (optional)
-
-```bash
-# Create a directory for file uploads
-mkdir -p /tmp/fileserver
-
-# Use any HTTP server that supports POST for file uploads
-# Example: Python's built-in server
-python3 -m http.server 8080 --directory /tmp/fileserver
-```
-
-### Step 3: Run Test Scenarios
-
-```bash
-# Scenario 1: Command & Control (JavaScript sender)
-node test/scenario1_command_control.js
-
-# Scenario 2: Large Arrow Table (JavaScript sender)
-node test/scenario2_large_table.js
-
-# Scenario 3: Julia-to-Julia communication
-# Run both Julia and JavaScript versions
-julia test/scenario3_julia_to_julia.jl
-node test/scenario3_julia_to_julia.js
-```
-
-## Usage
-
-### Scenario 1: Command & Control (Small JSON)
-
-#### JavaScript (Sender)
-```javascript
-const { SmartSend } = require('./js_bridge');
-
-const config = {
-    step_size: 0.01,
-    iterations: 1000
-};
-
-await SmartSend("control", config, "json", {
-    correlationId: "unique-id"
-});
-```
-
-#### Julia (Receiver)
-```julia
-using NATS
-using JSON3
-
-# Subscribe to control subject
-subscribe(nats, "control") do msg
-    env = MessageEnvelope(String(msg.data))
-    config = JSON3.read(env.payload)
-    
-    # Execute simulation with parameters
-    step_size = config.step_size
-    iterations = config.iterations
-    
-    # Send acknowledgment
-    response = Dict("status" => "Running", "correlation_id" => env.correlation_id)
-    publish(nats, "control_response", JSON3.stringify(response))
-end
-```
-
-### Scenario 2: Deep Dive Analysis (Large Arrow Table)
-
-#### Julia (Sender)
-```julia
-using Arrow
-using DataFrames
-
-# Create large DataFrame
-df = DataFrame(
-    id = 1:10_000_000,
-    value = rand(10_000_000),
-    category = rand(["A", "B", "C"], 10_000_000)
-)
-
-# Send via SmartSend with type="table"
-await SmartSend("analysis_results", df, "table");
-```
-
-#### JavaScript (Receiver)
-```javascript
-const { SmartReceive } = require('./js_bridge');
-
-const result = await SmartReceive(msg);
-
-// Use table data for visualization with Perspective.js or D3
-const table = result.data;
-```
-
-### Scenario 3: Live Binary Processing
-
-#### JavaScript (Sender)
-```javascript
-const { SmartSend } = require('./js_bridge');
-
-// Capture binary chunk
-const binaryData = await navigator.mediaDevices.getUserMedia({ binary: true });
-
-await SmartSend("binary_input", binaryData, "binary", {
-    metadata: {
-        sample_rate: 44100,
-        channels: 1
-    }
-});
-```
-
-#### Julia (Receiver)
-```julia
-using WAV
-using DSP
-
-# Receive binary data
-function process_binary(data)
-    # Perform FFT or AI transcription
-    spectrum = fft(data)
-    
-    # Send results back (JSON + Arrow table)
-    results = Dict("transcription" => "sample text", "spectrum" => spectrum)
-    await SmartSend("binary_output", results, "json")
-end
-```
-
-### Scenario 4: Catch-Up (JetStream)
-
-#### Julia (Producer)
-```julia
-using NATS
-
-function publish_health_status(nats)
-    jetstream = JetStream(nats, "health_updates")
-    
-    while true
-        status = Dict("cpu" => rand(), "memory" => rand())
-        publish(jetstream, "health", status)
-        sleep(5)  # Every 5 seconds
-    end
-end
-```
-
-#### JavaScript (Consumer)
-```javascript
-const { connect } = require('nats');
-
-const nc = await connect({ servers: ['nats://localhost:4222'] });
-const js = nc.jetstream();
-
-// Request replay from last 10 minutes
-const consumer = await js.pullSubscribe("health", { 
-    durable_name: "catchup",
-    max_batch: 100,
-    max_ack_wait: 30000
-});
-
-// Process historical and real-time messages
-for await (const msg of consumer) {
-    const result = await SmartReceive(msg);
-    // Process the data
-    msg.ack();
-}
-```
-
-## Configuration
-
-### Environment Variables
-
-| Variable | Default | Description |
-|----------|---------|-------------|
-| `NATS_URL` | `nats://localhost:4222` | NATS server URL |
-| `FILESERVER_URL` | `http://localhost:8080/upload` | HTTP file server URL |
-| `SIZE_THRESHOLD` | `1_000_000` | Size threshold in bytes (1MB) |
-
-### Message Envelope Schema
-
-```json
-{
-  "correlation_id": "uuid-v4-string",
-  "type": "json|table|binary",
-  "transport": "direct|link",
-  "payload": "base64-encoded-string",  // Only if transport=direct
-  "url": "http://fileserver/path/to/data",  // Only if transport=link
-  "metadata": {
-    "content_type": "application/octet-stream",
-    "content_length": 123456,
-    "format": "arrow_ipc_stream"
-  }
-}
-```
-
-## Performance Considerations
-
-### Zero-Copy Reading
-- Use Arrow's memory-mapped file reading
-- Avoid unnecessary data copying during deserialization
-- Use Apache Arrow's native IPC reader
-
-### Exponential Backoff
-- Maximum retry count: 5
-- Base delay: 100ms, max delay: 5000ms
-- Implemented in both Julia and JavaScript implementations
-
-### Correlation ID Logging
-- Log correlation_id at every stage
-- Include: send, receive, serialize, deserialize
-- Use structured logging format
-
-## Testing
-
-Run the test scripts:
-
-```bash
-# Scenario 1: Command & Control (JavaScript sender)
-node test/scenario1_command_control.js
-
-# Scenario 2: Large Arrow Table (JavaScript sender)
-node test/scenario2_large_table.js
-```
-
-## Troubleshooting
-
-### Common Issues
-
-1. **NATS Connection Failed**
-   - Ensure NATS server is running
-   - Check NATS_URL configuration
-
-2. **HTTP Upload Failed**
-   - Ensure file server is running
-   - Check FILESERVER_URL configuration
-   - Verify upload permissions
-
-3. **Arrow IPC Deserialization Error**
-   - Ensure data is properly serialized to Arrow format
-   - Check Arrow version compatibility
-
-## License
-
-MIT

docs/architecture.md

@@ -0,0 +1,555 @@
+# Architecture Documentation: Bi-Directional Data Bridge
+
+## Overview
+
+This document describes the architecture for a high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
+
+The system enables seamless communication for Julia applications:
+- **Julia** messaging with NATS
+
+### File Server Handler Architecture
+
+The system uses **handler functions** to abstract file server operations, allowing support for different file server implementations (e.g., Plik, AWS S3, custom HTTP server).
+
+**Handler Function Signatures:**
+
+```julia
+# Upload handler - uploads data to file server and returns URL
+# The handler is passed to smartsend as fileserver_upload_handler parameter
+# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8})
+# Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url"
+fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+
+# Download handler - fetches data from file server URL with exponential backoff
+# The handler is passed to smartreceive as fileserver_download_handler parameter
+# It receives: (url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)
+# Returns: Vector{UInt8} (the downloaded data)
+fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8}
+```
+
+This design allows the system to support multiple file server backends without changing the core messaging logic.
+
+### Multi-Payload Support (Standard API)
+
+The system uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.**
+
+**API Standard:**
+```julia
+# Input format for smartsend (always a list of tuples with type info)
+[(dataname1, data1, type1), (dataname2, data2, type2), ...]
+
+# Output format for smartreceive (returns a dictionary-like object with payloads field containing list of tuples)
+# Returns: Dict-like object with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}}
+# {
+#   "correlation_id": "...",
+#   "msg_id": "...",
+#   "timestamp": "...",
+#   "send_to": "...",
+#   "msg_purpose": "...",
+#   "sender_name": "...",
+#   "sender_id": "...",
+#   "receiver_name": "...",
+#   "receiver_id": "...",
+#   "reply_to": "...",
+#   "reply_to_msg_id": "...",
+#   "broker_url": "...",
+#   "metadata": {...},
+#   "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...]
+# }
+```
+
+**Supported Types:**
+- `"text"` - Plain text
+- `"dictionary"` - JSON-serializable dictionaries (Dict, NamedTuple)
+- `"table"` - Tabular data (DataFrame, array of structs)
+- `"image"` - Image data (Bitmap, PNG/JPG bytes)
+- `"audio"` - Audio data (WAV, MP3 bytes)
+- `"video"` - Video data (MP4, AVI bytes)
+- `"binary"` - Generic binary data (Vector{UInt8})
+
+This design allows per-payload type specification, enabling **mixed-content messages** where different payloads can use different serialization formats in a single message.
+
+**Examples:**
+
+```julia
+# Single payload - still wrapped in a list
+smartsend(
+    "/test",
+    [("dataname1", data1, "dictionary")],  # List with one tuple (data, type)
+    broker_url="nats://localhost:4222",
+    fileserver_upload_handler=plik_oneshot_upload
+)
+
+# Multiple payloads in one message with different types
+smartsend(
+    "/test",
+    [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")],
+    broker_url="nats://localhost:4222",
+    fileserver_upload_handler=plik_oneshot_upload
+)
+
+# Mixed content (e.g., chat with text, image, audio)
+smartsend(
+    "/chat",
+    [
+        ("message_text", "Hello!", "text"),
+        ("user_image", image_data, "image"),
+        ("audio_clip", audio_data, "audio")
+    ],
+    broker_url="nats://localhost:4222"
+)
+
+# Receive returns a dictionary envelope with all metadata and deserialized payloads
+env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000)
+# env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...]
+# env["correlation_id"], env["msg_id"], etc.
+# env is a dictionary containing envelope metadata and payloads field
+```
+
+## Architecture Diagram
+
+```mermaid
+flowchart TD
+    subgraph Client
+        App[Julia Application]
+    end
+
+    subgraph Server
+        Julia[Julia Service]
+        NATS[NATS Server]
+        FileServer[HTTP File Server]
+    end
+
+    App -->|NATS| NATS
+    NATS -->|NATS| Julia
+    Julia -->|NATS| NATS
+    Julia -->|HTTP POST| FileServer
+
+    style App fill:#e8f5e9
+    style Julia fill:#e8f5e9
+    style NATS fill:#fff3e0
+    style FileServer fill:#f3e5f5
+```
+
+## System Components
+
+### 1. msg_envelope_v1 - Message Envelope
+
+The `msg_envelope_v1` structure provides a comprehensive message format for bidirectional communication in Julia applications.
+
+**Julia Structure:**
+```julia
+struct msg_envelope_v1
+  correlation_id::String       # Unique identifier to track messages across systems
+  msg_id::String               # This message id
+  timestamp::String            # Message published timestamp
+  
+  send_to::String              # Topic/subject the sender sends to
+  msg_purpose::String          # Purpose of this message (ACK | NACK | updateStatus | shutdown | ...)
+  sender_name::String          # Sender name (e.g., "agent-wine-web-frontend")
+  sender_id::String            # Sender id (uuid4)
+  receiver_name::String        # Message receiver name (e.g., "agent-backend")
+  receiver_id::String          # Message receiver id (uuid4 or nothing for broadcast)
+  reply_to::String             # Topic to reply to
+  reply_to_msg_id::String      # Message id this message is replying to
+  broker_url::String           # NATS server address
+  
+  metadata::Dict{String, Any}
+  payloads::Vector{msg_payload_v1}  # Multiple payloads stored here
+end
+```
+
+**JSON Schema:**
+```json
+{
+  "correlation_id": "uuid-v4-string",
+  "msg_id": "uuid-v4-string",
+  "timestamp": "2024-01-15T10:30:00Z",
+  
+  "send_to": "topic/subject",
+  "msg_purpose": "ACK | NACK | updateStatus | shutdown | chat",
+  "sender_name": "agent-wine-web-frontend",
+  "sender_id": "uuid4",
+  "receiver_name": "agent-backend",
+  "receiver_id": "uuid4",
+  "reply_to": "topic",
+  "reply_to_msg_id": "uuid4",
+  "broker_url": "nats://localhost:4222",
+  
+  "metadata": {
+
+  },
+  
+  "payloads": [
+    {
+      "id": "uuid4",
+      "dataname": "login_image",
+      "payload_type": "image",
+      "transport": "direct",
+      "encoding": "base64",
+      "size": 15433,
+      "data": "base64-encoded-string",
+      "metadata": {
+
+      }
+    },
+    {
+      "id": "uuid4",
+      "dataname": "large_data",
+      "payload_type": "table",
+      "transport": "link",
+      "encoding": "none",
+      "size": 524288,
+      "data": "http://localhost:8080/file/UPLOAD_ID/FILE_ID/data.arrow",
+      "metadata": {
+
+      }
+    }
+  ]
+}
+```
+
+### 2. msg_payload_v1 - Payload Structure
+
+The `msg_payload_v1` structure provides flexible payload handling for various data types.
+
+**Julia Structure:**
+```julia
+struct msg_payload_v1
+  id::String                    # Id of this payload (e.g., "uuid4")
+  dataname::String              # Name of this payload (e.g., "login_image")
+  payload_type::String          # "text | dictionary | table | image | audio | video | binary"
+  transport::String             # "direct | link"
+  encoding::String              # "none | json | base64 | arrow-ipc"
+  size::Integer                 # Data size in bytes
+  data::Any                     # Payload data in case of direct transport or a URL in case of link
+  metadata::Dict{String, Any}   # Dict("checksum" => "sha256_hash", ...)
+end
+```
+
+**Key Features:**
+- Supports multiple data types: text, dictionary, table, image, audio, video, binary
+- Flexible transport: "direct" (NATS) or "link" (HTTP fileserver)
+- Multiple payloads per message (essential for chat with mixed content)
+- Per-payload and per-envelope metadata support
+
+### 3. Transport Strategy Decision Logic
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                     smartsend Function                      │
+│  Accepts: [(dataname1, data1, type1), ...]                  │
+│  (Type is per payload, not standalone)                      │
+└─────────────────────────────────────────────────────────────┘
+                             │
+                             ▼
+┌─────────────────────────────────────────────────────────────┐
+│  For each payload:                                          │
+│  1. Extract type from tuple                                 │
+│  2. Serialize based on type                                 │
+│  3. Check payload size                                      │
+└─────────────────────────────────────────────────────────────┘
+                             │
+            ┌────────────────┴─-────────────────┐
+            ▼                                   ▼
+      ┌─────────────────┐                 ┌─────────────────┐
+      │  Direct Path    │                 │  Link Path      │
+      │  (< 1MB)        │                 │  (> 1MB)        │
+      │                 │                 │                 │
+      │ • Serialize to  │                 │ • Serialize to  │
+      │   IOBuffer      │                 │   IOBuffer      │
+      │ • Base64 encode │                 │ • Upload to     │
+      │ • Publish to    │                 │   HTTP Server   │
+      │   NATS          │                 │ • Publish to    │
+      │   (with payload │                 │   NATS with URL │
+      │    in envelope) │                 │   (in envelope) │
+      └─────────────────┘                 └─────────────────┘
+```
+
+### 4. Julia Module Architecture
+
+```mermaid
+graph TD
+    subgraph JuliaModule
+        JuliaSmartSend[smartsend]
+        SizeCheck[Size Check]
+        DirectPath[Direct Path]
+        LinkPath[Link Path]
+        HTTPClient[HTTP Client]
+    end
+
+    JuliaSmartSend --> SizeCheck
+    SizeCheck -->|< 1MB| DirectPath
+    SizeCheck -->|>= 1MB| LinkPath
+    LinkPath --> HTTPClient
+
+    style JuliaModule fill:#c5e1a5
+```
+
+## Implementation Details
+
+### Julia Implementation
+
+#### Dependencies
+- `NATS.jl` - Core NATS functionality
+- `Arrow.jl` - Arrow IPC serialization
+- `JSON3.jl` - JSON parsing
+- `HTTP.jl` - HTTP client for file server
+- `Dates.jl` - Timestamps for logging
+
+#### smartsend Function
+
+```julia
+function smartsend(
+  subject::String,
+  data::AbstractArray{Tuple{String, Any, String}, 1};  # List of (dataname, data, type) tuples
+  broker_url::String = DEFAULT_BROKER_URL,  # NATS server URL
+  fileserver_url = DEFAULT_FILESERVER_URL,
+  fileserver_upload_handler::Function = plik_oneshot_upload,
+  size_threshold::Int = DEFAULT_SIZE_THRESHOLD,
+  correlation_id::Union{String, Nothing} = nothing,
+  msg_purpose::String = "chat",
+  sender_name::String = "NATSBridge",
+  receiver_name::String = "",
+  receiver_id::String = "",
+  reply_to::String = "",
+  reply_to_msg_id::String = "",
+  is_publish::Bool = true,   # Whether to automatically publish to NATS
+  NATS_connection::Union{NATS.Connection, Nothing} = nothing  # Pre-existing NATS connection (optional, saves connection overhead)
+)
+```
+
+**Keyword Parameter - NATS_connection:**
+- `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios where connection reuse provides performance benefits.
+
+**Connection Handling Logic:**
+```julia
+if is_publish == false
+  # skip publish a message
+elseif is_publish == true && NATS_connection === nothing
+  publish_message(broker_url, subject, env_json_str, cid)  # Creates new connection
+elseif is_publish == true && NATS_connection !== nothing
+  publish_message(NATS_connection, subject, env_json_str, cid)  # Uses provided connection
+end
+```
+
+**Return Value:**
+- Returns a tuple `(env, env_json_str)` where:
+  - `env::msg_envelope_v1` - The envelope object containing all metadata and payloads
+  - `env_json_str::String` - JSON string representation of the envelope for publishing
+
+**Options:**
+- `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern.
+
+The envelope object can be accessed directly for programmatic use, while the JSON string can be published directly to NATS using the request-reply pattern.
+
+**Input Format:**
+- `data::AbstractArray{Tuple{String, Any, String}}` - **Must be a list of (dataname, data, type) tuples**: `[("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...]`
+- Even for single payloads: `[(dataname1, data1, "type1")]`
+- Each payload can have a different type, enabling mixed-content messages
+
+**Flow:**
+1. Iterate through the list of `(dataname, data, type)` tuples
+2. For each payload: extract the type from the tuple and serialize accordingly
+3. Check payload size
+4. If < threshold: publish directly to NATS with Base64-encoded payload
+5. If >= threshold: upload to HTTP server, publish NATS with URL
+
+#### smartreceive Handler
+
+```julia
+function smartreceive(
+    msg::NATS.Msg;
+    fileserver_download_handler::Function = _fetch_with_backoff,
+    max_retries::Int = 5,
+    base_delay::Int = 100,
+    max_delay::Int = 5000
+)
+    # Parse envelope
+    # Iterate through all payloads
+    # For each payload: check transport type
+    #   If direct: decode Base64 payload
+    #   If link: fetch from URL with exponential backoff using fileserver_download_handler
+    # Deserialize payload based on type
+    # Return envelope dictionary with all metadata and deserialized payloads
+end
+```
+
+**Output Format:**
+- Returns a JSON object (dictionary) containing all envelope fields:
+  - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url`
+  - `metadata` - Message-level metadata dictionary
+  - `payloads` - List of tuples, each containing `(dataname, data, type)` with deserialized payload data
+
+**Process Flow:**
+1. Parse the JSON envelope to extract all fields
+2. Iterate through each payload in `payloads`
+3. For each payload:
+   - Determine transport type (`direct` or `link`)
+   - If `direct`: decode Base64 data from the message
+   - If `link`: fetch data from URL using exponential backoff (via `fileserver_download_handler`)
+   - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.)
+4. Return envelope dictionary with `payloads` field containing list of `(dataname, data, type)` tuples
+
+**Note:** The `fileserver_download_handler` receives `(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)` and returns `Vector{UInt8}`.
+
+#### publish_message Function
+
+The `publish_message` function provides two overloads for publishing messages to NATS:
+
+**Overload 1 - URL-based publishing (creates new connection):**
+```julia
+function publish_message(broker_url::String, subject::String, message::String, correlation_id::String)
+  conn = NATS.connect(broker_url)  # Create NATS connection
+  publish_message(conn, subject, message, correlation_id)
+end
+```
+
+**Overload 2 - Connection-based publishing (uses pre-existing connection):**
+```julia
+function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String)
+  try
+    NATS.publish(conn, subject, message)  # Publish message to NATS
+    log_trace(correlation_id, "Message published to $subject")  # Log successful publish
+  finally
+    NATS.drain(conn)  # Ensure connection is closed properly
+  end
+end
+```
+
+**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. This is a Julia-specific optimization that leverages function overloading.
+
+**Integration with smartsend:**
+```julia
+# When NATS_connection is provided to smartsend, it uses the connection-based publish_message
+env, env_json_str = smartsend(
+    "my.subject",
+    [("data", payload_data, "type")],
+    NATS_connection=my_connection,  # Pre-existing connection
+    is_publish=true
+)
+# Uses: publish_message(NATS_connection, subject, env_json_str, cid)
+
+# When NATS_connection is not provided, it uses the URL-based publish_message
+env, env_json_str = smartsend(
+    "my.subject",
+    [("data", payload_data, "type")],
+    broker_url="nats://localhost:4222",
+    is_publish=true
+)
+# Uses: publish_message(broker_url, subject, env_json_str, cid)
+```
+
+## Scenario Implementations
+
+### Scenario 1: Command & Control (Small Dictionary)
+
+**Julia (Sender/Receiver):**
+```julia
+# Subscribe to control subject
+# Parse JSON envelope
+# Execute simulation with parameters
+# Send acknowledgment
+```
+
+### Scenario 2: Deep Dive Analysis (Large Arrow Table)
+
+**Julia (Sender/Receiver):**
+```julia
+# Create large DataFrame
+# Convert to Arrow IPC stream
+# Check size (> 1MB)
+# Upload to HTTP server
+# Publish NATS with URL
+```
+
+### Scenario 3: Live Audio Processing
+
+**Julia (Sender/Receiver):**
+```julia
+# Receive audio data
+# Perform FFT or AI transcription
+# Send results back (JSON + Arrow table)
+```
+
+### Scenario 4: Catch-Up (JetStream)
+
+**Julia (Producer/Consumer):**
+```julia
+# Publish to JetStream
+# Include metadata for temporal tracking
+```
+
+### Scenario 5: Selection (Low Bandwidth)
+
+**Focus:** Small Arrow tables. The Action: Julia wants to send a small DataFrame to show on a receiving application for the user to choose.
+
+**Julia (Sender/Receiver):**
+```julia
+# Create small DataFrame (e.g., 50KB - 500KB)
+# Convert to Arrow IPC stream
+# Check payload size (< 1MB threshold)
+# Publish directly to NATS with Base64-encoded payload
+# Include metadata for dashboard selection context
+```
+
+### Scenario 6: Chat System
+
+**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging.
+
+**Multi-Payload Support:** The system supports mixed-payload messages where a single message can contain multiple payloads with different transport strategies. The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type.
+
+**Julia (Sender/Receiver):**
+```julia
+# Build chat message with mixed payloads:
+# - Text: direct transport (Base64)
+# - Small images: direct transport (Base64)
+# - Large images: link transport (HTTP URL)
+# - Audio/video: link transport (HTTP URL)
+# - Tables: direct or link depending on size
+# - Files: link transport (HTTP URL)
+# 
+# Each payload uses appropriate transport strategy:
+# - Size < 1MB → direct (NATS + Base64)
+# - Size >= 1MB → link (HTTP upload + NATS URL)
+# 
+# Include claim-check metadata for delivery tracking
+# Support bidirectional messaging with replyTo fields
+```
+
+**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components.
+
+**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads.
+
+## Performance Considerations
+
+### Zero-Copy Reading
+- Use Arrow's memory-mapped file reading
+- Avoid unnecessary data copying during deserialization
+- Use Apache Arrow's native IPC reader
+
+### Exponential Backoff
+- Implement exponential backoff for HTTP link fetching
+- Maximum retry count: 5
+- Base delay: 100ms, max delay: 5000ms
+
+### Correlation ID Logging
+- Log correlation_id at every stage
+- Include: send, receive, serialize, deserialize
+- Use structured logging format
+
+## Testing Strategy
+
+### Unit Tests
+- Test smartsend with various payload sizes
+- Test smartreceive with direct and link transport
+- Test Arrow IPC serialization/deserialization
+
+### Integration Tests
+- Test full flow with NATS server
+- Test large data transfer (> 100MB)
+- Test audio processing pipeline
+
+### Performance Tests
+- Measure throughput for small payloads
+- Measure throughput for large payloads

docs/implementation.md

@@ -0,0 +1,648 @@
+# Implementation Guide: Bi-Directional Data Bridge
+
+## Overview
+
+This document describes the implementation of the high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
+
+The system enables seamless communication for Julia applications.
+
+### Implementation Files
+
+NATSBridge is implemented in Julia:
+
+| Language | Implementation File | Description |
+|----------|---------------------|-------------|
+| **Julia** | [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Full Julia implementation with Arrow IPC support |
+
+### File Server Handler Architecture
+
+The system uses **handler functions** to abstract file server operations, allowing support for different file server implementations (e.g., Plik, AWS S3, custom HTTP server).
+
+**Handler Function Signatures:**
+
+```julia
+# Upload handler - uploads data to file server and returns URL
+# The handler is passed to smartsend as fileserver_upload_handler parameter
+# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8})
+# Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url"
+fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+
+# Download handler - fetches data from file server URL with exponential backoff
+# The handler is passed to smartreceive as fileserver_download_handler parameter
+# It receives: (url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)
+# Returns: Vector{UInt8} (the downloaded data)
+fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8}
+```
+
+This design allows the system to support multiple file server backends without changing the core messaging logic.
+
+### Multi-Payload Support (Standard API)
+
+The system uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.**
+
+**API Standard:**
+```julia
+# Input format for smartsend (always a list of tuples with type info)
+[(dataname1, data1, type1), (dataname2, data2, type2), ...]
+
+# Output format for smartreceive (returns a dictionary with payloads field containing list of tuples)
+# Returns: Dict with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}}
+# {
+#   "correlation_id": "...",
+#   "msg_id": "...",
+#   "timestamp": "...",
+#   "send_to": "...",
+#   "msg_purpose": "...",
+#   "sender_name": "...",
+#   "sender_id": "...",
+#   "receiver_name": "...",
+#   "receiver_id": "...",
+#   "reply_to": "...",
+#   "reply_to_msg_id": "...",
+#   "broker_url": "...",
+#   "metadata": {...},
+#   "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...]
+# }
+```
+
+**Supported Types:**
+- `"text"` - Plain text
+- `"dictionary"` - JSON-serializable dictionaries (Dict, NamedTuple)
+- `"table"` - Tabular data (DataFrame, array of structs)
+- `"image"` - Image data (Bitmap, PNG/JPG bytes)
+- `"audio"` - Audio data (WAV, MP3 bytes)
+- `"video"` - Video data (MP4, AVI bytes)
+- `"binary"` - Generic binary data (Vector{UInt8})
+
+This design allows per-payload type specification, enabling **mixed-content messages** where different payloads can use different serialization formats in a single message.
+
+**Examples:**
+```julia
+# Single payload - still wrapped in a list
+smartsend(
+    "/test",
+    [("dataname1", data1, "dictionary")],  # List with one tuple (data, type)
+    broker_url="nats://localhost:4222",
+    fileserver_upload_handler=plik_oneshot_upload
+)
+
+# Multiple payloads in one message with different types
+smartsend(
+    "/test",
+    [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")],
+    broker_url="nats://localhost:4222",
+    fileserver_upload_handler=plik_oneshot_upload
+)
+
+# Mixed content (e.g., chat with text, image, audio)
+smartsend(
+    "/chat",
+    [
+        ("message_text", "Hello!", "text"),
+        ("user_image", image_data, "image"),
+        ("audio_clip", audio_data, "audio")
+    ],
+    broker_url="nats://localhost:4222"
+)
+
+# Receive returns a dictionary envelope with all metadata and deserialized payloads
+env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000)
+# env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...]
+# env["correlation_id"], env["msg_id"], etc.
+# env is a dictionary containing envelope metadata and payloads field
+```
+
+## Architecture
+
+The Julia implementation follows the Claim-Check pattern:
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                         SmartSend Function                              │
+└─────────────────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────────────────┐
+│  Is payload size < 1MB?                                                 │
+└─────────────────────────────────────────────────────────────────────────┘
+                              │
+            ┌─────────────────┴─────────────────┐
+            ▼                                   ▼
+      ┌─────────────────┐                 ┌─────────────────┐
+      │  Direct Path    │                 │  Link Path      │
+      │  (< 1MB)        │                 │  (> 1MB)        │
+      │                 │                 │                 │
+      │ • Serialize to  │                 │ • Serialize to  │
+      │   Buffer        │                 │   Buffer        │
+      │ • Base64 encode │                 │ • Upload to     │
+      │ • Publish to    │                 │   HTTP Server   │
+      │   NATS          │                 │ • Publish to    │
+      │                 │                 │   NATS with URL │
+      └─────────────────┘                 └─────────────────┘
+```
+
+## smartsend Return Value
+
+The `smartsend` function now returns a tuple containing both the envelope object and the JSON string representation:
+
+```julia
+env, env_json_str = smartsend(...)
+# env::msg_envelope_v1    - The envelope object with all metadata and payloads
+# env_json_str::String    - JSON string for publishing to NATS
+```
+
+**Options:**
+- `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern.
+
+This enables two use cases:
+1. **Programmatic envelope access**: Access envelope fields directly via the `env` object
+2. **Direct JSON publishing**: Publish the JSON string directly using NATS request-reply pattern
+
+### Julia Module: [`src/NATSBridge.jl`](../src/NATSBridge.jl)
+
+The Julia implementation provides:
+
+- **[`msg_envelope_v1`](src/NATSBridge.jl)**: Struct for the unified JSON envelope
+- **[`msg_payload_v1`](src/NATSBridge.jl)**: Struct for individual payload representation
+- **[`smartsend()`](src/NATSBridge.jl)**: Handles transport selection based on payload size
+- **[`smartreceive()`](src/NATSBridge.jl)**: Handles both direct and link transport
+
+## Installation
+
+### Julia Dependencies
+
+```julia
+using Pkg
+Pkg.add("NATS")
+Pkg.add("Arrow")
+Pkg.add("JSON3")
+Pkg.add("HTTP")
+Pkg.add("UUIDs")
+Pkg.add("Dates")
+```
+
+## Usage Tutorial
+
+### Step 1: Start NATS Server
+
+```bash
+docker run -p 4222:4222 nats:latest
+```
+
+### Step 2: Start HTTP File Server (optional)
+
+```bash
+# Create a directory for file uploads
+mkdir -p /tmp/fileserver
+
+# Use any HTTP server that supports POST for file uploads
+# Example: Python's built-in server
+python3 -m http.server 8080 --directory /tmp/fileserver
+```
+
+### Step 3: Run Test Scenarios
+
+```bash
+# Scenario 1: Command & Control
+julia test/scenario1_command_control.jl
+
+# Scenario 2: Large Arrow Table
+julia test/scenario2_large_table.jl
+
+# Scenario 3: Julia-to-Julia communication
+julia test/scenario3_julia_to_julia.jl
+```
+
+## Usage
+
+### Scenario 1: Command & Control (Small Dictionary)
+
+**Focus:** Sending small dictionary configurations. This is the simplest use case for command and control scenarios.
+
+**Julia (Sender/Receiver):**
+```julia
+using NATSBridge
+
+# Send small dictionary config (wrapped in list with type)
+config = Dict("step_size" => 0.01, "iterations" => 1000, "threshold" => 0.5)
+env, env_json_str = smartsend(
+    "control",
+    [("config", config, "dictionary")],
+    broker_url="nats://localhost:4222"
+)
+# env: msg_envelope_v1 with all metadata and payloads
+# env_json_str: JSON string for publishing
+```
+
+**Julia (Sender/Receiver) with NATS_connection for connection reuse:**
+```julia
+using NATSBridge
+
+# Create connection once for high-frequency publishing
+conn = NATS.connect("nats://localhost:4222")
+
+# Send multiple messages using the same connection (saves connection overhead)
+for i in 1:100
+    config = Dict("iteration" => i, "data" => rand())
+    smartsend(
+        "control",
+        [("config", config, "dictionary")],
+        NATS_connection=conn,  # Reuse connection
+        is_publish=true
+    )
+end
+
+# Close connection when done
+NATS.close(conn)
+```
+
+**Use Case:** High-frequency publishing scenarios where connection reuse provides performance benefits by avoiding the overhead of establishing a new NATS connection for each message.
+
+### Basic Multi-Payload Example
+
+#### Julia (Sender)
+```julia
+using NATSBridge
+
+# Send multiple payloads in one message (type is required per payload)
+smartsend(
+    "/test",
+    [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")],
+    broker_url="nats://localhost:4222",
+    fileserver_url="http://localhost:8080"
+)
+
+# Even single payload must be wrapped in a list with type
+smartsend("/test", [("single_data", mydata, "dictionary")], broker_url="nats://localhost:4222")
+```
+
+#### Julia (Receiver)
+```julia
+using NATSBridge
+
+# Receive returns a dictionary with envelope metadata and payloads field
+env = smartreceive(msg)
+# env["payloads"] = [(dataname1, data1, "dictionary"), (dataname2, data2, "table"), ...]
+```
+
+### Scenario 2: Deep Dive Analysis (Large Arrow Table)
+
+#### Julia (Sender)
+```julia
+using Arrow
+using DataFrames
+
+# Create large DataFrame
+df = DataFrame(
+    id = 1:10_000_000,
+    value = rand(10_000_000),
+    category = rand(["A", "B", "C"], 10_000_000)
+)
+
+# Send via smartsend - wrapped in list with type
+# Large payload will use link transport (HTTP fileserver)
+env, env_json_str = smartsend(
+    "analysis_results",
+    [("table_data", df, "table")],
+    broker_url="nats://localhost:4222",
+    fileserver_url="http://localhost:8080"
+)
+# env: msg_envelope_v1 with all metadata and payloads
+# env_json_str: JSON string for publishing
+```
+
+#### smartsend Function Signature (Julia)
+
+```julia
+function smartsend(
+  subject::String,
+  data::AbstractArray{Tuple{String, Any, String}, 1};  # List of (dataname, data, type) tuples
+  broker_url::String = DEFAULT_BROKER_URL,  # NATS server URL
+  fileserver_url = DEFAULT_FILESERVER_URL,
+  fileserver_upload_handler::Function = plik_oneshot_upload,
+  size_threshold::Int = DEFAULT_SIZE_THRESHOLD,
+  correlation_id::Union{String, Nothing} = nothing,
+  msg_purpose::String = "chat",
+  sender_name::String = "NATSBridge",
+  receiver_name::String = "",
+  receiver_id::String = "",
+  reply_to::String = "",
+  reply_to_msg_id::String = "",
+  is_publish::Bool = true,
+  NATS_connection::Union{NATS.Connection, Nothing} = nothing  # Pre-existing NATS connection (optional)
+)
+```
+
+**New Keyword Parameter:**
+- `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios.
+
+**Connection Handling Logic:**
+```julia
+if is_publish == false
+  # skip publish
+elseif is_publish == true && NATS_connection === nothing
+  publish_message(broker_url, subject, env_json_str, cid)  # Creates new connection
+elseif is_publish == true && NATS_connection !== nothing
+  publish_message(NATS_connection, subject, env_json_str, cid)  # Uses provided connection
+end
+```
+
+**Example with pre-existing connection:**
+```julia
+using NATSBridge
+
+# Create connection once
+conn = NATS.connect("nats://localhost:4222")
+
+# Send multiple messages using the same connection
+for i in 1:100
+    data = rand(1000)
+    smartsend(
+        "analysis_results",
+        [("table_data", data, "table")],
+        NATS_connection=conn,  # Reuse connection
+        is_publish=true
+    )
+end
+
+# Close connection when done
+NATS.close(conn)
+```
+
+#### publish_message Function
+
+The `publish_message` function provides two overloads for publishing messages to NATS:
+
+**Overload 1 - URL-based publishing (creates new connection):**
+```julia
+function publish_message(broker_url::String, subject::String, message::String, correlation_id::String)
+  conn = NATS.connect(broker_url)  # Create NATS connection
+  publish_message(conn, subject, message, correlation_id)
+end
+```
+
+**Overload 2 - Connection-based publishing (uses pre-existing connection):**
+```julia
+function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String)
+  try
+    NATS.publish(conn, subject, message)  # Publish message to NATS
+    log_trace(correlation_id, "Message published to $subject")
+  finally
+    NATS.drain(conn)  # Ensure connection is closed properly
+  end
+end
+```
+
+**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish.
+
+**Integration with smartsend:**
+```julia
+# When NATS_connection is provided to smartsend, it uses the connection-based publish_message
+env, env_json_str = smartsend(
+    "my.subject",
+    [("data", payload_data, "type")],
+    NATS_connection=my_connection,  # Pre-existing connection
+    is_publish=true
+)
+# Uses: publish_message(NATS_connection, subject, env_json_str, cid)
+
+# When NATS_connection is not provided, it uses the URL-based publish_message
+env, env_json_str = smartsend(
+    "my.subject",
+    [("data", payload_data, "type")],
+    broker_url="nats://localhost:4222",
+    is_publish=true
+)
+# Uses: publish_message(broker_url, subject, env_json_str, cid)
+```
+
+**API Consistency Note:**
+- **Julia:** Uses `NATS_connection` keyword parameter with function overloading for automatic connection management
+
+### Scenario 3: Live Binary Processing
+
+**Julia (Sender/Receiver):**
+```julia
+using NATSBridge
+
+# Binary data wrapped in list with type
+smartsend(
+    "binary_input",
+    [("audio_chunk", binary_buffer, "binary")],
+    broker_url="nats://localhost:4222",
+    metadata=["sample_rate" => 44100, "channels" => 1]
+)
+```
+
+### Scenario 4: Catch-Up (JetStream)
+
+**Julia (Producer/Consumer):**
+```julia
+using NATSBridge
+
+function publish_health_status(broker_url)
+    # Send status wrapped in list with type
+    status = Dict("cpu" => rand(), "memory" => rand())
+    env, env_json_str = smartsend(
+        "health",
+        [("status", status, "dictionary")],
+        broker_url=broker_url
+    )
+    sleep(5)  # Every 5 seconds
+end
+```
+
+### Scenario 5: Selection (Low Bandwidth)
+
+**Focus:** Small Arrow tables. The Action: Julia wants to send a small DataFrame to show on a receiving application for the user to choose.
+
+**Julia (Sender/Receiver):**
+```julia
+using NATSBridge
+using DataFrames
+
+# Create small DataFrame (e.g., 50KB - 500KB)
+options_df = DataFrame(
+    id = 1:10,
+    name = ["Option A", "Option B", "Option C", "Option D", "Option E",
+            "Option F", "Option G", "Option H", "Option I", "Option J"],
+    description = ["Description A", "Description B", "Description C", "Description D", "Description E",
+                   "Description F", "Description G", "Description H", "Description I", "Description J"]
+)
+
+# Convert to Arrow IPC stream
+# Check payload size (< 1MB threshold)
+# Publish directly to NATS with Base64-encoded payload
+# Include metadata for dashboard selection context
+env, env_json_str = smartsend(
+    "dashboard.selection",
+    [("options_table", options_df, "table")],
+    broker_url="nats://localhost:4222",
+    metadata=Dict("context" => "user_selection")
+)
+# env: msg_envelope_v1 with all metadata and payloads
+# env_json_str: JSON string for publishing
+```
+
+**Use Case:** Julia server generates a list of available options (e.g., file selections, configuration presets) as a small DataFrame and sends to a receiving application for user selection. The selection is then sent back to Julia for processing.
+
+### Scenario 6: Chat System
+
+**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging.
+
+**Multi-Payload Support:** The system supports mixed-payload messages where a single message can contain multiple payloads with different transport strategies. The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type.
+
+**Julia (Sender/Receiver):**
+```julia
+using NATSBridge
+
+# Build chat message with mixed payloads:
+# - Text: direct transport (Base64)
+# - Small images: direct transport (Base64)
+# - Large images: link transport (HTTP URL)
+# - Audio/video: link transport (HTTP URL)
+# - Tables: direct or link depending on size
+# - Files: link transport (HTTP URL)
+# 
+# Each payload uses appropriate transport strategy:
+# - Size < 1MB → direct (NATS + Base64)
+# - Size >= 1MB → link (HTTP upload + NATS URL)
+# 
+# Include claim-check metadata for delivery tracking
+# Support bidirectional messaging with replyTo fields
+
+# Example: Chat with text, small image, and large file
+chat_message = [
+    ("message_text", "Hello, this is a test message!", "text"),
+    ("user_avatar", image_bytes, "image"),  # Small image, direct transport
+    ("large_document", large_file_bytes, "binary")  # Large file, link transport
+]
+
+env, env_json_str = smartsend(
+    "chat.room123",
+    chat_message,
+    broker_url="nats://localhost:4222",
+    msg_purpose="chat",
+    reply_to="chat.room123.responses"
+)
+# env: msg_envelope_v1 with all metadata and payloads
+# env_json_str: JSON string for publishing
+```
+
+**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components.
+
+**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads.
+
+## Configuration
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `NATS_URL` | `nats://localhost:4222` | NATS server URL |
+| `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL (base URL without `/upload` suffix) |
+| `SIZE_THRESHOLD` | `1_000_000` | Size threshold in bytes (1MB) |
+
+### Message Envelope Schema
+
+```json
+{
+  "correlation_id": "uuid-v4-string",
+  "msg_id": "uuid-v4-string",
+  "timestamp": "2024-01-15T10:30:00Z",
+  
+  "send_to": "topic/subject",
+  "msg_purpose": "ACK | NACK | updateStatus | shutdown | chat",
+  "sender_name": "agent-wine-web-frontend",
+  "sender_id": "uuid4",
+  "receiver_name": "agent-backend",
+  "receiver_id": "uuid4",
+  "reply_to": "topic",
+  "reply_to_msg_id": "uuid4",
+  "broker_url": "nats://localhost:4222",
+  
+  "metadata": {
+    "content_type": "application/octet-stream",
+    "content_length": 123456
+  },
+  
+  "payloads": [
+    {
+      "id": "uuid4",
+      "dataname": "login_image",
+      "payload_type": "image",
+      "transport": "direct",
+      "encoding": "base64",
+      "size": 15433,
+      "data": "base64-encoded-string",
+      "metadata": {
+        "checksum": "sha256_hash"
+      }
+    }
+  ]
+}
+```
+
+## Performance Considerations
+
+### Zero-Copy Reading
+- Use Arrow's memory-mapped file reading
+- Avoid unnecessary data copying during deserialization
+- Use Apache Arrow's native IPC reader
+
+### Exponential Backoff
+- Maximum retry count: 5
+- Base delay: 100ms, max delay: 5000ms
+
+### Correlation ID Logging
+- Log correlation_id at every stage
+- Include: send, receive, serialize, deserialize
+- Use structured logging format
+
+## Testing
+
+Run the test scripts for Julia:
+
+### Julia Tests
+
+```bash
+# Text message exchange
+julia test/test_julia_to_julia_text_sender.jl
+julia test/test_julia_to_julia_text_receiver.jl
+
+# Dictionary exchange
+julia test/test_julia_to_julia_dict_sender.jl
+julia test/test_julia_to_julia_dict_receiver.jl
+
+# File transfer
+julia test/test_julia_to_julia_file_sender.jl
+julia test/test_julia_to_julia_file_receiver.jl
+
+# Mixed payload types
+julia test/test_julia_to_julia_mix_payloads_sender.jl
+julia test/test_julia_to_julia_mix_payloads_receiver.jl
+
+# Table exchange
+julia test/test_julia_to_julia_table_sender.jl
+julia test/test_julia_to_julia_table_receiver.jl
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **NATS Connection Failed**
+   - Ensure NATS server is running
+
+2. **HTTP Upload Failed**
+   - Ensure file server is running
+   - Check `fileserver_url` configuration
+   - Verify upload permissions
+
+3. **Arrow IPC Deserialization Error**
+   - Ensure data is properly serialized to Arrow format
+   - Check Arrow version compatibility
+
+## License
+
+MIT

etc.jl

@@ -1,42 +1,9 @@
+Task: Update README.md to reflect recent changes in NATSbridge package.
 
-""" fileServerURL = "http://192.168.88.104:8080"
-filepath = "/home/ton/docker-apps/sendreceive/image/test.zip"
-filename = basename(filepath)
-filebytes = read(filepath)
+Context: the package has been updated with the NATS_connection keyword and the publish_message function.
 
-plik_oneshot_upload - Upload a single file to a plik server using one-shot mode
+Requirements:
 
-This function uploads a raw byte array to a plik server in one-shot mode (no upload session).
-It first creates a one-shot upload session by sending a POST request with `{"OneShot": true}`,
-retrieves an upload ID and token, then uploads the file data as multipart form data using the token.
-
-The function handles the entire flow:
-1. Obtains an upload ID and token from the server
-2. Uploads the provided binary data as a file using the `X-UploadToken` header
-3. Returns identifiers and download URL for the uploaded file
-
-# Arguments:
-  - `fileServerURL::String` - Base URL of the plik server (e.g., `"http://192.168.88.104:8080"`)
-  - `filename::String` - Name of the file being uploaded
-  - `data::Vector{UInt8}` - Raw byte data of the file content
-
-# Return:
-  - A named tuple with fields:
-    - `uploadid::String` - ID of the one-shot upload session
-    - `fileid::String` - ID of the uploaded file within the session
-    - `downloadurl::String` - Full URL to download the uploaded file
-
-# Example
-```jldoctest
-using HTTP, JSON
-
-# Example data: "Hello" as bytes
-data = collect("Hello World!" |> collect |> CodeUnits |> collect)
-
-# Upload to local plik server
-result = plik_oneshot_upload("http://192.168.88.104:8080", "hello.txt", data)
-
-# Download URL for the uploaded file
-println(result.downloadurl)
-```
-"""
+Source of Truth: Treat the updated NATSbridge code as the definitive source. Update README.md to align exactly with these changes.
+API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages.
+Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency.

examples/tutorial.md

@@ -0,0 +1,304 @@
+# NATSBridge Tutorial
+
+A step-by-step guide to get started with NATSBridge - a high-performance, bi-directional data bridge for **Julia**.
+
+## Table of Contents
+
+1. [Overview](#overview)
+2. [Prerequisites](#prerequisites)
+3. [Installation](#installation)
+4. [Quick Start](#quick-start)
+5. [Basic Examples](#basic-examples)
+6. [Advanced Usage](#advanced-usage)
+
+---
+
+## Overview
+
+NATSBridge enables seamless communication for Julia applications through NATS, with automatic transport selection based on payload size:
+
+- **Direct Transport**: Payloads < 1MB are sent directly via NATS (Base64 encoded)
+- **Link Transport**: Payloads >= 1MB are uploaded to an HTTP file server and referenced via URL
+
+### Supported Payload Types
+
+| Type | Description |
+|------|-------------|
+| `text` | Plain text strings |
+| `dictionary` | JSON-serializable dictionaries |
+| `table` | Tabular data (Arrow IPC format) |
+| `image` | Image data (PNG, JPG bytes) |
+| `audio` | Audio data (WAV, MP3 bytes) |
+| `video` | Video data (MP4, AVI bytes) |
+| `binary` | Generic binary data |
+
+---
+
+## Prerequisites
+
+Before you begin, ensure you have:
+
+1. **NATS Server** running (or accessible)
+2. **HTTP File Server** (optional, for large payloads > 1MB)
+3. **Julia** with required packages
+
+---
+
+## Installation
+
+### Julia
+
+```julia
+using Pkg
+Pkg.add("NATS")
+Pkg.add("Arrow")
+Pkg.add("JSON3")
+Pkg.add("HTTP")
+Pkg.add("UUIDs")
+Pkg.add("Dates")
+```
+
+---
+
+## Quick Start
+
+### Step 1: Start NATS Server
+
+```bash
+docker run -p 4222:4222 nats:latest
+```
+
+### Step 2: Start HTTP File Server (Optional)
+
+```bash
+# Create a directory for file uploads
+mkdir -p /tmp/fileserver
+
+# Use any HTTP server that supports POST for file uploads
+python3 -m http.server 8080 --directory /tmp/fileserver
+```
+
+### Step 3: Send Your First Message
+
+#### Julia
+
+```julia
+using NATSBridge
+
+# Send a text message
+data = [("message", "Hello World", "text")]
+env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222")
+# env: msg_envelope_v1 object with all metadata and payloads
+# env_json_str: JSON string representation of the envelope for publishing
+println("Message sent!")
+
+# Or use is_publish=false to get envelope and JSON without publishing
+env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=false)
+# env: msg_envelope_v1 object
+# env_json_str: JSON string for publishing to NATS
+```
+
+### Step 4: Receive Messages
+
+#### Julia
+
+```julia
+using NATSBridge
+
+# Receive and process message
+env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff)
+for (dataname, data, type) in env["payloads"]
+    println("Received $dataname: $data")
+end
+```
+
+---
+
+## Basic Examples
+
+### Example 1: Sending a Dictionary
+
+#### Julia
+
+```julia
+using NATSBridge
+
+config = Dict(
+    "wifi_ssid" => "MyNetwork",
+    "wifi_password" => "password123",
+    "update_interval" => 60
+)
+
+data = [("config", config, "dictionary")]
+env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222")
+```
+
+### Example 2: Sending Binary Data (Image)
+
+#### Julia
+
+```julia
+using NATSBridge
+
+# Read image file
+image_data = read("image.png")
+
+data = [("user_image", image_data, "binary")]
+env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222")
+```
+
+### Example 3: Request-Response Pattern
+
+#### Julia (Requester)
+
+```julia
+using NATSBridge
+
+# Send command with reply-to
+data = [("command", Dict("action" => "read_sensor"), "dictionary")]
+env, env_json_str = smartsend(
+    "/device/command",
+    data,
+    broker_url="nats://localhost:4222",
+    reply_to="/device/response",
+    reply_to_msg_id="cmd-001"
+)
+# env: msg_envelope_v1 object
+# env_json_str: JSON string for publishing to NATS
+```
+
+#### Julia (Responder)
+
+```julia
+using NATS, NATSBridge
+
+# Configuration
+const SUBJECT = "/device/command"
+const NATS_URL = "nats://localhost:4222"
+
+function test_responder()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff)
+        
+        # Extract reply_to from the envelope metadata
+        reply_to = env["reply_to"]
+        
+        for (dataname, data, type) in env["payloads"]
+            if dataname == "command" && data["action"] == "read_sensor"
+                response = Dict("sensor_id" => "sensor-001", "value" => 42.5)
+                # Send response to the reply_to subject from the request
+                if !isempty(reply_to)
+                    smartsend(reply_to, [("data", response, "dictionary")])
+                end
+            end
+        end
+    end
+    
+    sleep(120)
+    NATS.drain(conn)
+end
+
+test_responder()
+```
+
+---
+
+## Advanced Usage
+
+### Example 4: Large Payloads (File Server)
+
+For payloads larger than 1MB, NATSBridge automatically uses the file server:
+
+#### Julia
+
+```julia
+using NATSBridge
+
+# Create large data (> 1MB)
+large_data = rand(UInt8, 2_000_000)
+
+env, env_json_str = smartsend(
+    "/data/large",
+    [("large_file", large_data, "binary")],
+    broker_url="nats://localhost:4222",
+    fileserver_url="http://localhost:8080"
+)
+
+# The envelope will contain the download URL
+println("File uploaded to: $(env.payloads[1].data)")
+```
+
+### Example 5: Mixed Content (Chat with Text + Image)
+
+NATSBridge supports sending multiple payloads with different types in a single message:
+
+#### Julia
+
+```julia
+using NATSBridge
+
+image_data = read("avatar.png")
+
+data = [
+    ("message_text", "Hello with image!", "text"),
+    ("user_avatar", image_data, "image")
+]
+
+env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222")
+```
+
+### Example 6: Table Data (Arrow IPC)
+
+For tabular data, NATSBridge uses Apache Arrow IPC format:
+
+#### Julia
+
+```julia
+using NATSBridge
+using DataFrames
+
+# Create DataFrame
+df = DataFrame(
+    id = [1, 2, 3],
+    name = ["Alice", "Bob", "Charlie"],
+    score = [95, 88, 92]
+)
+
+data = [("students", df, "table")]
+env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222")
+```
+
+---
+
+## Next Steps
+
+1. **Explore the test directory** for more examples
+2. **Check the documentation** for advanced configuration options
+
+---
+
+## Troubleshooting
+
+### Connection Issues
+
+- Ensure NATS server is running: `docker ps | grep nats`
+- Check firewall settings
+- Verify NATS URL configuration
+
+### File Server Issues
+
+- Ensure file server is running and accessible
+- Check upload permissions
+- Verify file server URL configuration
+
+### Serialization Errors
+
+- Verify data type matches the specified type
+- Check that binary data is in the correct format (Vector{UInt8})
+
+---
+
+## License
+
+MIT

examples/walkthrough.md

@@ -0,0 +1,703 @@
+# NATSBridge Walkthrough
+
+A comprehensive guide to building real-world applications with NATSBridge.
+
+## Table of Contents
+
+1. [Introduction](#introduction)
+2. [Architecture Overview](#architecture-overview)
+3. [Building a Chat Application](#building-a-chat-application)
+4. [Building a File Transfer System](#building-a-file-transfer-system)
+5. [Building a Streaming Data Pipeline](#building-a-streaming-data-pipeline)
+6. [Performance Optimization](#performance-optimimization)
+7. [Best Practices](#best-practices)
+
+---
+
+## Introduction
+
+This walkthrough will guide you through building several real-world applications using NATSBridge. We'll cover:
+
+- Chat applications with rich media support
+- File transfer systems with claim-check pattern
+- Streaming data pipelines
+
+Each section builds on the previous one, gradually increasing in complexity.
+
+---
+
+## Architecture Overview
+
+### System Components
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                     NATSBridge Architecture                     │
+├─────────────────────────────────────────────────────────────────┤
+│  ┌──────────────┐    ┌──────────────┐                          │
+│  │   Julia      │    │  NATS         │                          │
+│  │  (NATS.jl)   │◄──►│  Server       │                          │
+│  └──────────────┘    └──────────────┘                          │
+│         │                      │                               │
+│         ▼                      ▼                               │
+│      ┌──────────────────────────────────────┐                 │
+│      │          File Server                  │                 │
+│      │         (HTTP Upload)                 │                 │
+│      └──────────────────────────────────────┘                 │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+### Message Flow
+
+1. **Sender** creates a message envelope with payloads
+2. **NATSBridge** serializes and encodes payloads
+3. **Transport Decision**: Small payloads go directly to NATS, large payloads are uploaded to file server
+4. **NATS** routes messages to subscribers
+5. **Receiver** fetches payloads (from NATS or file server)
+6. **NATSBridge** deserializes and decodes payloads
+
+---
+
+## Building a Chat Application
+
+Let's build a full-featured chat application that supports text, images, and file attachments.
+
+### Step 1: Set Up the Project
+
+```bash
+# Create project directory
+mkdir -p chat-app/src
+cd chat-app
+
+# Create configuration file
+cat > config.json << 'EOF'
+{
+  "nats_url": "nats://localhost:4222",
+  "fileserver_url": "http://localhost:8080",
+  "size_threshold": 1048576
+}
+EOF
+```
+
+### Step 2: Create the Chat Interface (Julia)
+
+```julia
+# src/chat_ui.jl
+using NATSBridge, NATS
+
+struct ChatUI
+    messages::Vector{Dict}
+    current_room::String
+end
+
+function ChatUI()
+    ChatUI(Dict[], "")
+end
+
+function send_message(ui::ChatUI, message_input::String, selected_file::Union{Nothing, String})
+    data = []
+    
+    # Add text message
+    if !isempty(message_input)
+        push!(data, ("text", message_input, "text"))
+    end
+    
+    # Add file if selected
+    if selected_file !== nothing
+        file_data = read(selected_file)
+        file_type = get_file_type(selected_file)
+        push!(data, ("attachment", file_data, file_type))
+    end
+    
+    return data
+end
+
+function get_file_type(filename::String)::String
+    if endswith(filename, ".png") || endswith(filename, ".jpg")
+        return "image"
+    elseif endswith(filename, ".mp3") || endswith(filename, ".wav")
+        return "audio"
+    elseif endswith(filename, ".mp4") || endswith(filename, ".avi")
+        return "video"
+    else
+        return "binary"
+    end
+end
+
+function add_message(ui::ChatUI, user::String, text::String, attachment::Union{Nothing, Dict})
+    push!(ui.messages, Dict(
+        "user" => user,
+        "text" => text,
+        "attachment" => attachment
+    ))
+end
+```
+
+### Step 3: Create the Message Handler
+
+```julia
+# src/chat_handler.jl
+using NATSBridge, NATS
+
+struct ChatHandler
+    nats::NATS.Connection
+    ui::ChatUI
+end
+
+function ChatHandler(nats_connection::NATS.Connection)
+    ChatHandler(nats_connection, ChatUI())
+end
+
+function start(handler::ChatHandler)
+    # Subscribe to chat rooms
+    rooms = ["general", "tech", "random"]
+    
+    for room in rooms
+        NATS.subscribe(handler.nats, "/chat/$room") do msg
+            handle_message(handler, msg)
+        end
+    end
+    
+    println("Chat handler started")
+end
+
+function handle_message(handler::ChatHandler, msg::NATS.Msg)
+    env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff)
+    
+    # Extract sender info from envelope
+    sender = get(env, "sender_name", "Anonymous")
+    
+    # Process each payload
+    for (dataname, data, type) in env["payloads"]
+        if type == "text"
+            add_message(handler.ui, sender, data, nothing)
+        elseif type == "image"
+            # Convert to data URL for display
+            base64_data = base64encode(data)
+            attachment = Dict(
+                "type" => "image",
+                "data" => "data:image/png;base64,$base64_data"
+            )
+            add_message(handler.ui, sender, "", attachment)
+        else
+            # For other types, use file server URL
+            attachment = Dict("type" => type, "data" => data)
+            add_message(handler.ui, sender, "", attachment)
+        end
+    end
+end
+
+function download_file(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8}
+    # Implement exponential backoff for file server downloads
+    # Return downloaded data as Vector{UInt8}
+end
+```
+
+### Step 4: Run the Application
+
+```bash
+# Start NATS
+docker run -p 4222:4222 nats:latest
+
+# Start file server
+mkdir -p /tmp/fileserver
+python3 -m http.server 8080 --directory /tmp/fileserver
+
+# Run chat app
+julia src/chat_ui.jl
+julia src/chat_handler.jl
+```
+
+---
+
+## Building a File Transfer System
+
+Let's build a file transfer system that handles large files efficiently.
+
+### Step 1: File Upload Service (Julia)
+
+```julia
+# src/file_upload_service.jl
+using NATSBridge, HTTP
+
+struct FileUploadService
+    broker_url::String
+    fileserver_url::String
+end
+
+function FileUploadService(broker_url::String, fileserver_url::String)
+    FileUploadService(broker_url, fileserver_url)
+end
+
+function upload_file(service::FileUploadService, file_path::String, recipient::String)::Dict
+    file_data = read(file_path)
+    file_name = basename(file_path)
+    
+    data = [("file", file_data, "binary")]
+    
+    env, env_json_str = smartsend(
+        "/files/$recipient",
+        data,
+        broker_url=service.broker_url,
+        fileserver_url=service.fileserver_url
+    )
+    
+    return env
+end
+
+function upload_large_file(service::FileUploadService, file_path::String, recipient::String)::Dict
+    file_size = stat(file_path).size
+    
+    if file_size > 100 * 1024 * 1024  # > 100MB
+        println("File too large for direct upload, using streaming...")
+        return stream_upload(service, file_path, recipient)
+    end
+    
+    return upload_file(service, file_path, recipient)
+end
+
+function stream_upload(service::FileUploadService, file_path::String, recipient::String)::Dict
+    # Implement streaming upload to file server
+    # This would require a more sophisticated file server
+    # For now, we'll use the standard upload
+    return upload_file(service, file_path, recipient)
+end
+```
+
+### Step 2: File Download Service (Julia)
+
+```julia
+# src/file_download_service.jl
+using NATSBridge
+
+struct FileDownloadService
+    nats_url::String
+end
+
+function FileDownloadService(nats_url::String)
+    FileDownloadService(nats_url)
+end
+
+function download_file(service::FileDownloadService, msg::NATS.Msg, sender::String, download_id::String)
+    # Subscribe to sender's file channel
+    env = smartreceive(msg, fileserver_download_handler=fetch_from_url)
+    
+    # Process each payload
+    for (dataname, data, type) in env["payloads"]
+        if type == "binary"
+            file_path = "/downloads/$dataname"
+            write(file_path, data)
+            println("File saved to $file_path")
+        end
+    end
+end
+
+function fetch_from_url(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8}
+    # Fetch data from URL with exponential backoff
+    # Return downloaded data as Vector{UInt8}
+end
+```
+
+### Step 3: File Transfer CLI (Julia)
+
+```julia
+# src/cli.jl
+using NATSBridge, Readlines, FileIO
+
+function main()
+    config = JSON3.read(read("config.json", String))
+    
+    println("File Transfer System")
+    println("====================")
+    println("1. Upload file")
+    println("2. Download file")
+    println("3. List pending downloads")
+    
+    print("Enter choice: ")
+    choice = readline()
+    
+    if choice == "1"
+        upload_file_cli(config)
+    elseif choice == "2"
+        download_file_cli(config)
+    end
+end
+
+function upload_file_cli(config)
+    print("Enter file path: ")
+    file_path = readline()
+    
+    print("Enter recipient: ")
+    recipient = readline()
+    
+    file_service = FileUploadService(config.nats_url, config.fileserver_url)
+    
+    try
+        env = upload_file(file_service, file_path, recipient)
+        println("Upload successful!")
+        println("File ID: $(env["payloads"][1][1])")
+    catch error
+        println("Upload failed: $(error)")
+    end
+end
+
+function download_file_cli(config)
+    print("Enter sender: ")
+    sender = readline()
+    
+    file_service = FileDownloadService(config.nats_url)
+    
+    try
+        download_file(file_service, sender)
+        println("Download complete!")
+    catch error
+        println("Download failed: $(error)")
+    end
+end
+
+main()
+```
+
+---
+
+## Building a Streaming Data Pipeline
+
+Let's build a data pipeline that processes streaming data from sensors.
+
+### Step 1: Sensor Data Model (Julia)
+
+```julia
+# src/sensor_data.jl
+using Dates, DataFrames
+
+struct SensorReading
+    sensor_id::String
+    timestamp::String
+    value::Float64
+    unit::String
+    metadata::Dict{String, Any}
+end
+
+function SensorReading(sensor_id::String, value::Float64, unit::String, metadata::Dict{String, Any}=Dict())
+    SensorReading(
+        sensor_id,
+        ISODateTime(now(), Dates.Second) |> string,
+        value,
+        unit,
+        metadata
+    )
+end
+
+struct SensorBatch
+    readings::Vector{SensorReading}
+end
+
+function SensorBatch()
+    SensorBatch(SensorReading[])
+end
+
+function add_reading(batch::SensorBatch, reading::SensorReading)
+    push!(batch.readings, reading)
+end
+
+function to_dataframe(batch::SensorBatch)::DataFrame
+    data = Dict{String, Any}()
+    data["sensor_id"] = [r.sensor_id for r in batch.readings]
+    data["timestamp"] = [r.timestamp for r in batch.readings]
+    data["value"] = [r.value for r in batch.readings]
+    data["unit"] = [r.unit for r in batch.readings]
+    
+    return DataFrame(data)
+end
+```
+
+### Step 2: Sensor Sender (Julia)
+
+```julia
+# src/sensor_sender.jl
+using NATSBridge, Dates, Random
+
+struct SensorSender
+    broker_url::String
+    fileserver_url::String
+end
+
+function SensorSender(broker_url::String, fileserver_url::String)
+    SensorSender(broker_url, fileserver_url)
+end
+
+function send_reading(sender::SensorSender, sensor_id::String, value::Float64, unit::String)
+    reading = SensorReading(sensor_id, value, unit)
+    
+    data = [("reading", reading.metadata, "dictionary")]
+    
+    # Default: is_publish=True (automatically publishes to NATS)
+    smartsend(
+        "/sensors/$sensor_id",
+        data,
+        broker_url=sender.broker_url,
+        fileserver_url=sender.fileserver_url
+    )
+end
+
+function prepare_message_only(sender::SensorSender, sensor_id::String, value::Float64, unit::String)
+    """Prepare a message without publishing (is_publish=False)."""
+    reading = SensorReading(sensor_id, value, unit)
+    
+    data = [("reading", reading.metadata, "dictionary")]
+    
+    # With is_publish=False, returns (env, env_json_str) without publishing
+    env, env_json_str = smartsend(
+        "/sensors/$sensor_id/prepare",
+        data,
+        broker_url=sender.broker_url,
+        fileserver_url=sender.fileserver_url,
+        is_publish=false
+    )
+    
+    # Now you can publish manually using NATS request-reply pattern
+    # nc.request(subject, env_json_str, reply_to=reply_to_topic)
+    
+    return env, env_json_str
+end
+
+function send_batch(sender::SensorSender, readings::Vector{SensorReading})
+    batch = SensorBatch()
+    for reading in readings
+        add_reading(batch, reading)
+    end
+    
+    df = to_dataframe(batch)
+    
+    # Convert to Arrow IPC format
+    import Arrow
+    table = Arrow.Table(df)
+    
+    # Serialize to Arrow IPC
+    import IOBuffer
+    buf = IOBuffer()
+    Arrow.write(buf, table)
+    
+    arrow_data = take!(buf)
+    
+    # Send based on size
+    if length(arrow_data) < 1048576  # < 1MB
+        data = [("batch", arrow_data, "table")]
+        smartsend(
+            "/sensors/batch",
+            data,
+            broker_url=sender.broker_url,
+            fileserver_url=sender.fileserver_url
+        )
+    else
+        # Upload to file server
+        data = [("batch", arrow_data, "table")]
+        smartsend(
+            "/sensors/batch",
+            data,
+            broker_url=sender.broker_url,
+            fileserver_url=sender.fileserver_url
+        )
+    end
+end
+```
+
+### Step 3: Sensor Receiver (Julia)
+
+```julia
+# src/sensor_receiver.jl
+using NATSBridge, Arrow, DataFrames, IOBuffer
+
+struct SensorReceiver
+    fileserver_download_handler::Function
+end
+
+function SensorReceiver(download_handler::Function)
+    SensorReceiver(download_handler)
+end
+
+function process_reading(receiver::SensorReceiver, msg::NATS.Msg)
+    env = smartreceive(msg, receiver.fileserver_download_handler)
+    
+    for (dataname, data, data_type) in env["payloads"]
+        if data_type == "dictionary"
+            # Process dictionary payload
+            println("Received: $dataname = $data")
+        elseif data_type == "table"
+            # Deserialize Arrow IPC
+            buf = IOBuffer(data)
+            table = Arrow.read(buf)
+            df = DataFrame(table)
+            println("Received batch with $(nrow(df)) readings")
+            println(df)
+        end
+    end
+end
+```
+
+---
+
+## Performance Optimization
+
+### 1. Batch Processing
+
+```julia
+# Batch multiple readings into a single message
+function send_batch_readings(sender::SensorSender, readings::Vector{Tuple{String, Float64, String}})
+    batch = SensorBatch()
+    
+    for (sensor_id, value, unit) in readings
+        reading = SensorReading(sensor_id, value, unit)
+        add_reading(batch, reading)
+    end
+    
+    df = to_dataframe(batch)
+    
+    # Convert to Arrow IPC
+    import Arrow
+    table = Arrow.Table(df)
+    
+    # Serialize to Arrow IPC
+    import IOBuffer
+    buf = IOBuffer()
+    Arrow.write(buf, table)
+    
+    arrow_data = take!(buf)
+    
+    # Send as single message
+    smartsend(
+        "/sensors/batch",
+        [("batch", arrow_data, "table")],
+        broker_url=sender.broker_url
+    )
+end
+```
+
+### 2. Connection Reuse
+
+```julia
+# Reuse NATS connections
+function create_connection_pool()
+    connections = Dict{String, NATS.Connection}()
+    
+    function get_connection(nats_url::String)::NATS.Connection
+        if !haskey(connections, nats_url)
+            connections[nats_url] = NATS.connect(nats_url)
+        end
+        return connections[nats_url]
+    end
+    
+    function close_all()
+        for conn in values(connections)
+            NATS.drain(conn)
+        end
+        empty!(connections)
+    end
+    
+    return (get_connection= get_connection, close_all=close_all)
+end
+```
+
+### 3. Caching
+
+```julia
+# Cache file server responses
+using Base.Threads
+
+const file_cache = Dict{String, Vector{UInt8}}()
+
+function fetch_with_caching(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8}
+    if haskey(file_cache, url)
+        return file_cache[url]
+    end
+    
+    # Fetch from file server
+    data = _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id)
+    
+    # Cache the result
+    file_cache[url] = data
+    
+    return data
+end
+```
+
+---
+
+## Best Practices
+
+### 1. Error Handling
+
+```julia
+function safe_smartsend(subject::String, data::Vector{Tuple}, kwargs...)
+    try
+        return smartsend(subject, data; kwargs...)
+    catch error
+        println("Failed to send message: $(error)")
+        return nothing
+    end
+end
+```
+
+### 2. Logging
+
+```julia
+using Logging
+
+function log_send(subject::String, data::Vector{Tuple}, correlation_id::String)
+    @info "Sending to $subject: $(length(data)) payloads, correlation_id=$correlation_id"
+end
+
+function log_receive(correlation_id::String, num_payloads::Int)
+    @info "Received message: $num_payloads payloads, correlation_id=$correlation_id"
+end
+```
+
+### 3. Rate Limiting
+
+```julia
+using Dates, Collections
+
+struct RateLimiter
+    max_requests::Int
+    time_window::Float64
+    requests::Deque{Float64}
+end
+
+function RateLimiter(max_requests::Int, time_window::Float64)
+    RateLimiter(max_requests, time_window, Deque{Float64}())
+end
+
+function allow(limiter::RateLimiter)::Bool
+    now = time()
+    
+    # Remove old requests
+    while !isempty(limiter.requests) && limiter.requests[1] < now - limiter.time_window
+        popfirst!(limiter.requests)
+    end
+    
+    if length(limiter.requests) >= limiter.max_requests
+        return false
+    end
+    
+    push!(limiter.requests, now)
+    return true
+end
+```
+
+---
+
+## Conclusion
+
+This walkthrough covered:
+
+- Building a chat application with rich media support
+- Building a file transfer system with claim-check pattern
+- Building a streaming data pipeline for sensor data
+
+For more information, check the [API documentation](../src/README.md) and [test examples](../test/).
+
+---
+
+## License
+
+MIT

package.json

@@ -0,0 +1,28 @@
+{
+	"name": "natsbridge",
+	"version": "1.0.0",
+	"description": "Bi-Directional Data Bridge for JavaScript using NATS",
+	"main": "src/NATSBridge.js",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1",
+		"lint": "eslint src/*.js test/*.js"
+	},
+	"keywords": [
+		"nats",
+		"message-broker",
+		"bridge",
+		"arrow",
+		"serialization"
+	],
+	"author": "",
+	"license": "MIT",
+	"dependencies": {
+		"nats": "^2.9.0",
+		"apache-arrow": "^14.0.0",
+		"uuid": "^9.0.0"
+	},
+	"devDependencies": {
+		"eslint": "^8.0.0",
+		"jest": "^29.0.0"
+	}
+}

plik_fileserver/docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  plik:
+    image: rootgg/plik:latest
+    container_name: plik-server
+    restart: unless-stopped
+    ports:
+      - "8080:8080"
+    volumes:
+      # # Mount the config file (created below)
+      # - ./plikd.cfg:/home/plik/server/plikd.cfg
+      # Mount local folder for uploads and database
+      - ./plik-data:/data
+    # Set user to match your host UID to avoid permission issues
+    user: "1000:1000"

src/NATSBridge.js

@@ -1,245 +0,0 @@
-/**
- * Bi-Directional Data Bridge - JavaScript Module
- * Implements SmartSend and SmartReceive for NATS communication
- */
-
-const { v4: uuidv4 } = require('uuid');
-const { decode, encode } = require('base64-url');
-const Arrow = require('apache-arrow');
-
-// Constants
-const DEFAULT_SIZE_THRESHOLD = 1_000_000; // 1MB
-const DEFAULT_NATS_URL = 'nats://localhost:4222';
-const DEFAULT_FILESERVER_URL = 'http://localhost:8080/upload';
-
-// Logging helper
-function logTrace(correlationId, message) {
-    const timestamp = new Date().toISOString();
-    console.log(`[${timestamp}] [Correlation: ${correlationId}] ${message}`);
-}
-
-// Message Envelope Class
-class MessageEnvelope {
-    constructor(options = {}) {
-        this.correlation_id = options.correlation_id || uuidv4();
-        this.type = options.type || 'json';
-        this.transport = options.transport || 'direct';
-        this.payload = options.payload || null;
-        this.url = options.url || null;
-        this.metadata = options.metadata || {};
-    }
-
-    static fromJSON(jsonStr) {
-        const data = JSON.parse(jsonStr);
-        return new MessageEnvelope({
-            correlation_id: data.correlation_id,
-            type: data.type,
-            transport: data.transport,
-            payload: data.payload || null,
-            url: data.url || null,
-            metadata: data.metadata || {}
-        });
-    }
-
-    toJSON() {
-        const obj = {
-            correlation_id: this.correlation_id,
-            type: this.type,
-            transport: this.transport
-        };
-
-        if (this.payload) {
-            obj.payload = this.payload;
-        }
-
-        if (this.url) {
-            obj.url = this.url;
-        }
-
-        if (Object.keys(this.metadata).length > 0) {
-            obj.metadata = this.metadata;
-        }
-
-        return JSON.stringify(obj);
-    }
-}
-
-// SmartSend for JavaScript - Handles transport selection based on payload size
-async function SmartSend(subject, data, type = 'json', options = {}) {
-    const {
-        natsUrl = DEFAULT_NATS_URL,
-        fileserverUrl = DEFAULT_FILESERVER_URL,
-        sizeThreshold = DEFAULT_SIZE_THRESHOLD,
-        correlationId = uuidv4()
-    } = options;
-
-    logTrace(correlationId, `Starting SmartSend for subject: ${subject}`);
-
-    // Serialize data based on type
-    const payloadBytes = _serializeData(data, type, correlationId);
-    const payloadSize = payloadBytes.length;
-
-    logTrace(correlationId, `Serialized payload size: ${payloadSize} bytes`);
-
-    // Decision: Direct vs Link
-    if (payloadSize < sizeThreshold) {
-        // Direct path - Base64 encode and send via NATS
-        const payloadBase64 = encode(payloadBytes);
-        logTrace(correlationId, `Using direct transport for ${payloadSize} bytes`);
-
-        const env = new MessageEnvelope({
-            correlation_id: correlationId,
-            type: type,
-            transport: 'direct',
-            payload: payloadBase64,
-            metadata: {
-                content_length: payloadSize.toString(),
-                format: 'arrow_ipc_stream'
-            }
-        });
-
-        await publishMessage(natsUrl, subject, env.toJSON(), correlationId);
-        return env;
-    } else {
-        // Link path - Upload to HTTP server, send URL via NATS
-        logTrace(correlationId, `Using link transport, uploading to fileserver`);
-
-        const url = await uploadToServer(payloadBytes, fileserverUrl, correlationId);
-
-        const env = new MessageEnvelope({
-            correlation_id: correlationId,
-            type: type,
-            transport: 'link',
-            url: url,
-            metadata: {
-                content_length: payloadSize.toString(),
-                format: 'arrow_ipc_stream'
-            }
-        });
-
-        await publishMessage(natsUrl, subject, env.toJSON(), correlationId);
-        return env;
-    }
-}
-
-// Helper: Serialize data based on type
-function _serializeData(data, type, correlationId) {
-    if (type === 'json') {
-        const jsonStr = JSON.stringify(data);
-        return Buffer.from(jsonStr, 'utf8');
-    } else if (type === 'table') {
-        // Table data - convert to Arrow IPC stream
-        const writer = new Arrow.Writer();
-        writer.writeTable(data);
-        return writer.toByteArray();
-    } else if (type === 'binary') {
-        // Binary data - treat as binary
-        if (data instanceof Buffer) {
-            return data;
-        } else if (Array.isArray(data)) {
-            return Buffer.from(data);
-        } else {
-            throw new Error('Binary data must be binary (Buffer or Array)');
-        }
-    } else {
-        throw new Error(`Unknown type: ${type}`);
-    }
-}
-
-// Helper: Publish message to NATS
-async function publishMessage(natsUrl, subject, message, correlationId) {
-    const { connect } = require('nats');
-    
-    try {
-        const nc = await connect({ servers: [natsUrl] });
-        await nc.publish(subject, message);
-        logTrace(correlationId, `Message published to ${subject}`);
-        nc.close();
-    } catch (error) {
-        logTrace(correlationId, `Failed to publish message: ${error.message}`);
-        throw error;
-    }
-}
-
-// SmartReceive for JavaScript - Handles both direct and link transport
-async function SmartReceive(msg, options = {}) {
-    const {
-        fileserverUrl = DEFAULT_FILESERVER_URL,
-        maxRetries = 5,
-        baseDelay = 100,
-        maxDelay = 5000
-    } = options;
-
-    const env = MessageEnvelope.fromJSON(msg.data);
-
-    logTrace(env.correlation_id, `Processing received message`);
-
-    if (env.transport === 'direct') {
-        logTrace(env.correlation_id, `Direct transport - decoding payload`);
-
-        const payloadBytes = decode(env.payload);
-        const data = _deserializeData(payloadBytes, env.type, env.correlation_id, env.metadata);
-
-        return { data, envelope: env };
-    } else if (env.transport === 'link') {
-        logTrace(env.correlation_id, `Link transport - fetching from URL`);
-
-        const data = await _fetchWithBackoff(env.url, maxRetries, baseDelay, maxDelay, env.correlation_id);
-        const result = _deserializeData(data, env.type, env.correlation_id, env.metadata);
-
-        return { data: result, envelope: env };
-    } else {
-        throw new Error(`Unknown transport type: ${env.transport}`);
-    }
-}
-
-// Helper: Fetch with exponential backoff
-async function _fetchWithBackoff(url, maxRetries, baseDelay, maxDelay, correlationId) {
-    let delay = baseDelay;
-
-    for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-            const response = await fetch(url);
-            if (response.ok) {
-                const buffer = await response.arrayBuffer();
-                logTrace(correlationId, `Successfully fetched data from ${url} on attempt ${attempt}`);
-                return new Uint8Array(buffer);
-            } else {
-                throw new Error(`Failed to fetch: ${response.status}`);
-            }
-        } catch (error) {
-            logTrace(correlationId, `Attempt ${attempt} failed: ${error.message}`);
-
-            if (attempt < maxRetries) {
-                await new Promise(resolve => setTimeout(resolve, delay));
-                delay = Math.min(delay * 2, maxDelay);
-            }
-        }
-    }
-
-    throw new Error(`Failed to fetch data after ${maxRetries} attempts`);
-}
-
-// Helper: Deserialize data based on type
-async function _deserializeData(data, type, correlationId, metadata) {
-    if (type === 'json') {
-        const jsonStr = new TextDecoder().decode(data);
-        return JSON.parse(jsonStr);
-    } else if (type === 'table') {
-        // Deserialize Arrow IPC stream to Table
-        const table = Arrow.Table.from(data);
-        return table;
-    } else if (type === 'binary') {
-        // Return binary binary data
-        return data;
-    } else {
-        throw new Error(`Unknown type: ${type}`);
-    }
-}
-
-// Export functions
-module.exports = {
-    SmartSend,
-    SmartReceive,
-    MessageEnvelope
-};

test/scenario1_command_control.jl

@@ -1,67 +0,0 @@
-#!/usr/bin/env julia
-# Scenario 1: Command & Control (Small JSON)
-# Tests small JSON payloads (< 1MB) sent directly via NATS
-
-using NATS
-using JSON3
-using UUIDs
-
-# Include the bridge module
-include("../src/julia_bridge.jl")
-using .BiDirectionalBridge
-
-# Configuration
-const CONTROL_SUBJECT = "control"
-const RESPONSE_SUBJECT = "control_response"
-const NATS_URL = "nats://localhost:4222"
-
-# Create correlation ID for tracing
-correlation_id = string(uuid4())
-
-# Receiver: Listen for control commands
-function start_control_listener()
-    conn = NATS.Connection(NATS_URL)
-    try
-        NATS.subscribe(conn, CONTROL_SUBJECT) do msg
-            log_trace(msg.data)
-            
-            # Parse the envelope
-            env = MessageEnvelope(String(msg.data))
-            
-            # Parse JSON payload
-            config = JSON3.read(env.payload)
-            
-            # Execute simulation with parameters
-            step_size = config.step_size
-            iterations = config.iterations
-            
-            # Simulate processing
-            sleep(0.1)  # Simulate some work
-            
-            # Send acknowledgment
-            response = Dict(
-                "status" => "Running",
-                "correlation_id" => env.correlation_id,
-                "step_size" => step_size,
-                "iterations" => iterations
-            )
-            
-            NATS.publish(conn, RESPONSE_SUBJECT, JSON3.stringify(response))
-            log_trace("Sent response: $(JSON3.stringify(response))")
-        end
-        
-        # Keep listening for 5 seconds
-        sleep(5)
-    finally
-        NATS.close(conn)
-    end
-end
-
-# Helper: Log with correlation ID
-function log_trace(message)
-    timestamp = Dates.now()
-    println("[$timestamp] [Correlation: $correlation_id] $message")
-end
-
-# Run the listener
-start_control_listener()

test/scenario1_command_control.js

@@ -1,34 +0,0 @@
-#!/usr/bin/env node
-// Scenario 1: Command & Control (Small JSON)
-// Tests small JSON payloads (< 1MB) sent directly via NATS
-
-const { SmartSend } = require('../js_bridge');
-
-// Configuration
-const CONTROL_SUBJECT = "control";
-const NATS_URL = "nats://localhost:4222";
-
-// Create correlation ID for tracing
-const correlationId = require('uuid').v4();
-
-// Sender: Send control command to Julia
-async function sendControlCommand() {
-    const config = {
-        step_size: 0.01,
-        iterations: 1000
-    };
-
-    // Send via SmartSend with type="json"
-    const env = await SmartSend(
-        CONTROL_SUBJECT,
-        config,
-        "json",
-        { correlationId }
-    );
-
-    console.log(`Sent control command with correlation_id: ${correlationId}`);
-    console.log(`Envelope: ${JSON.stringify(env, null, 2)}`);
-}
-
-// Run the sender
-sendControlCommand().catch(console.error);

test/scenario2_large_table.jl

@@ -1,66 +0,0 @@
-#!/usr/bin/env julia
-# Scenario 2: Deep Dive Analysis (Large Arrow Table)
-# Tests large Arrow tables (> 1MB) sent via HTTP fileserver
-
-using NATS
-using Arrow
-using DataFrames
-using JSON3
-using UUIDs
-
-# Include the bridge module
-include("../src/julia_bridge.jl")
-using .BiDirectionalBridge
-
-# Configuration
-const ANALYSIS_SUBJECT = "analysis_results"
-const RESPONSE_SUBJECT = "analysis_response"
-const NATS_URL = "nats://localhost:4222"
-
-# Create correlation ID for tracing
-correlation_id = string(uuid4())
-
-# Receiver: Listen for analysis results
-function start_analysis_listener()
-    conn = NATS.Connection(NATS_URL)
-    try
-        NATS.subscribe(conn, ANALYSIS_SUBJECT) do msg
-            log_trace("Received message from $(msg.subject)")
-            
-            # Parse the envelope
-            env = MessageEnvelope(String(msg.data))
-            
-            # Use SmartReceive to handle the data
-            result = SmartReceive(msg)
-            
-            # Process the data based on type
-            if result.envelope.type == "table"
-                df = result.data
-                log_trace("Received DataFrame with $(nrows(df)) rows")
-                log_trace("DataFrame columns: $(names(df))")
-                
-                # Send acknowledgment
-                response = Dict(
-                    "status" => "Processed",
-                    "correlation_id" => env.correlation_id,
-                    "row_count" => nrows(df)
-                )
-                NATS.publish(conn, RESPONSE_SUBJECT, JSON3.stringify(response))
-            end
-        end
-        
-        # Keep listening for 10 seconds
-        sleep(10)
-    finally
-        NATS.close(conn)
-    end
-end
-
-# Helper: Log with correlation ID
-function log_trace(message)
-    timestamp = Dates.now()
-    println("[$timestamp] [Correlation: $correlation_id] $message")
-end
-
-# Run the listener
-start_analysis_listener()

test/scenario2_large_table.js

@@ -1,54 +0,0 @@
-#!/usr/bin/env node
-// Scenario 2: Deep Dive Analysis (Large Arrow Table)
-// Tests large Arrow tables (> 1MB) sent via HTTP fileserver
-
-const { SmartSend } = require('../js_bridge');
-
-// Configuration
-const ANALYSIS_SUBJECT = "analysis_results";
-const NATS_URL = "nats://localhost:4222";
-
-// Create correlation ID for tracing
-const correlationId = require('uuid').v4();
-
-// Sender: Send large Arrow table to Julia
-async function sendLargeTable() {
-    // Create a large DataFrame-like structure (10 million rows)
-    // For testing, we'll create a smaller but still large table
-    const numRows = 1000000; // 1 million rows
-    
-    const data = {
-        id: Array.from({ length: numRows }, (_, i) => i + 1),
-        value: Array.from({ length: numRows }, () => Math.random()),
-        category: Array.from({ length: numRows }, () => ['A', 'B', 'C'][Math.floor(Math.random() * 3)])
-    };
-    
-    // Convert to Arrow Table
-    const { Table, Vector, RecordBatch } = require('apache-arrow');
-    
-    const idVector = Vector.from(data.id);
-    const valueVector = Vector.from(data.value);
-    const categoryVector = Vector.from(data.category);
-    
-    const table = Table.from({
-        id: idVector,
-        value: valueVector,
-        category: categoryVector
-    });
-    
-    // Send via SmartSend with type="table"
-    const env = await SmartSend(
-        ANALYSIS_SUBJECT,
-        table,
-        "table",
-        { correlationId }
-    );
-    
-    console.log(`Sent large table with ${numRows} rows`);
-    console.log(`Correlation ID: ${correlationId}`);
-    console.log(`Transport: ${env.transport}`);
-    console.log(`URL: ${env.url || 'N/A'}`);
-}
-
-// Run the sender
-sendLargeTable().catch(console.error);

test/scenario3_julia_to_julia.jl

@@ -1,66 +0,0 @@
-#!/usr/bin/env julia
-# Scenario 3: Julia-to-Julia Service Communication
-# Tests bi-directional communication between two Julia services
-
-using NATS
-using Arrow
-using DataFrames
-using JSON3
-using UUIDs
-
-# Include the bridge module
-include("../src/julia_bridge.jl")
-using .BiDirectionalBridge
-
-# Configuration
-const SUBJECT1 = "julia_to_js"
-const SUBJECT2 = "js_to_julia"
-const RESPONSE_SUBJECT = "response"
-const NATS_URL = "nats://localhost:4222"
-
-# Create correlation ID for tracing
-correlation_id = string(uuid4())
-
-# Julia-to-Julia Test: Large Arrow Table
-function test_julia_to_julia_large_table()
-    conn = NATS.Connection(NATS_URL)
-    try
-        # Subscriber on SUBJECT2 to receive data from Julia sender
-        NATS.subscribe(conn, SUBJECT2) do msg
-            log_trace("[$(Dates.now())] Received on $SUBJECT2")
-            
-            # Use SmartReceive to handle the data
-            result = SmartReceive(msg)
-            
-            # Check transport type
-            if result.envelope.transport == "direct"
-                log_trace("Received direct transport with $(length(result.data)) bytes")
-            else
-                # For link transport, result.data is the URL
-                log_trace("Received link transport at $(result.data)")
-            end
-            
-            # Send response back
-            response = Dict(
-                "status" => "Processed",
-                "correlation_id" => result.envelope.correlation_id,
-                "timestamp" => Dates.now()
-            )
-            NATS.publish(conn, RESPONSE_SUBJECT, JSON3.stringify(response))
-        end
-        
-        # Keep listening
-        sleep(5)
-    finally
-        NATS.close(conn)
-    end
-end
-
-# Helper: Log with correlation ID
-function log_trace(message)
-    timestamp = Dates.now()
-    println("[$timestamp] [Correlation: $correlation_id] $message")
-end
-
-# Run the test
-test_julia_to_julia_large_table()

test/scenario_tests.md

@@ -1,148 +0,0 @@
-# Test Scenarios for Bi-Directional Data Bridge
-
-## Scenario 1: Command & Control (Small JSON)
-Tests small JSON payloads (< 1MB) sent directly via NATS.
-
-### Julia (Receiver)
-```julia
-using NATS
-using JSON3
-
-# Subscribe to control subject
-subscribe(nats, "control") do msg
-    env = MessageEnvelope(String(msg.data))
-    
-    # Parse JSON payload
-    config = JSON3.read(env.payload)
-    
-    # Execute simulation with parameters
-    step_size = config.step_size
-    iterations = config.iterations
-    
-    # Send acknowledgment
-    response = Dict("status" => "Running", "correlation_id" => env.correlation_id)
-    publish(nats, "control_response", JSON3.stringify(response))
-end
-```
-
-### JavaScript (Sender)
-```javascript
-const { SmartSend } = require('./js_bridge');
-
-// Create small JSON config
-const config = {
-    step_size: 0.01,
-    iterations: 1000
-};
-
-// Send via SmartSend with type="json"
-await SmartSend("control", config, "json");
-```
-
-## Scenario 2: Deep Dive Analysis (Large Arrow Table)
-Tests large Arrow tables (> 1MB) sent via HTTP fileserver.
-
-### Julia (Sender)
-```julia
-using Arrow
-using DataFrames
-
-# Create large DataFrame (500MB, 10 million rows)
-df = DataFrame(
-    id = 1:10_000_000,
-    value = rand(10_000_000),
-    category = rand(["A", "B", "C"], 10_000_000)
-)
-
-# Convert to Arrow IPC stream and send
-await SmartSend("analysis_results", df, "table");
-```
-
-### JavaScript (Receiver)
-```javascript
-const { SmartReceive } = require('./js_bridge');
-
-// Receive message with URL
-const result = await SmartReceive(msg);
-
-// Fetch data from HTTP server
-const table = result.data;
-
-// Load into Perspective.js or D3
-// Use table data for visualization
-```
-
-## Scenario 3: Live Binary Processing
-Tests binary data (binary) sent from JS to Julia for FFT/transcription.
-
-### JavaScript (Sender)
-```javascript
-const { SmartSend } = require('./js_bridge');
-
-// Capture binary chunk (2 seconds, 44.1kHz, 1 channel)
-const binaryData = await navigator.mediaDevices.getUserMedia({ binary: true });
-
-// Send as binary with metadata headers
-await SmartSend("binary_input", binaryData, "binary", {
-    metadata: {
-        sample_rate: 44100,
-        channels: 1
-    }
-});
-```
-
-### Julia (Receiver)
-```julia
-using WAV
-using DSP
-
-# Receive binary data
-function process_binary(data)
-    # Perform FFT or AI transcription
-    spectrum = fft(data)
-    
-    # Send results back (JSON + Arrow table)
-    results = Dict("transcription" => "sample text", "spectrum" => spectrum)
-    await SmartSend("binary_output", results, "json")
-end
-```
-
-## Scenario 4: Catch-Up (JetStream)
-Tests temporal decoupling with NATS JetStream.
-
-### Julia (Producer)
-```julia
-# Publish to JetStream
-using NATS
-
-function publish_health_status(nats)
-    jetstream = JetStream(nats, "health_updates")
-    
-    while true
-        status = Dict("cpu" => rand(), "memory" => rand())
-        publish(jetstream, "health", status)
-        sleep(5)  # Every 5 seconds
-    end
-end
-```
-
-### JavaScript (Consumer)
-```javascript
-const { connect } = require('nats');
-
-const nc = await connect({ servers: ['nats://localhost:4222'] });
-const js = nc.jetstream();
-
-// Request replay from last 10 minutes
-const consumer = await js.pullSubscribe("health", { 
-    durable_name: "catchup",
-    max_batch: 100,
-    max_ack_wait: 30000
-});
-
-// Process historical and real-time messages
-for await (const msg of consumer) {
-    const result = await SmartReceive(msg);
-    // Process the data
-    msg.ack();
-}

test/test_julia_dict_receiver.jl

@@ -0,0 +1,82 @@
+#!/usr/bin/env julia
+# Test script for Dictionary transport testing
+# Tests receiving 1 large and 1 small Dictionaries via direct and link transport
+# Uses NATSBridge.jl smartreceive with "dictionary" type
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_dict_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test dictionary transfer                                  #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+
+# Receiver: Listen for messages and verify Dictionary handling
+function test_dict_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Use NATSBridge.smartreceive to handle the data
+        # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay)
+        result = NATSBridge.smartreceive(
+            msg;
+            max_retries = 5,
+            base_delay = 100,
+            max_delay = 5000
+        )
+        
+        # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples
+        for (dataname, data, data_type) in result["payloads"]
+            if isa(data, JSON.Object{String, Any})
+                log_trace("Received Dictionary '$dataname' of type $data_type")
+                
+                # Display dictionary contents
+                println("  Contents:")
+                for (key, value) in data
+                    println("    $key => $value")
+                end
+                
+                # Save to JSON file
+                output_path = "./received_$dataname.json"
+                json_str = JSON.json(data, 2)
+                write(output_path, json_str)
+                log_trace("Saved Dictionary to $output_path")
+            else
+                log_trace("Received unexpected data type for '$dataname': $(typeof(data))")
+            end
+        end
+    end
+    
+    # Keep listening for 10 seconds
+    sleep(120)  
+    NATS.drain(conn)
+end
+
+
+# Run the test
+println("Starting Dictionary transport test...")
+println("Note: This receiver will wait for messages from the sender.")
+println("Run test_julia_to_julia_dict_sender.jl first to send test data.")
+
+# Run receiver
+println("testing smartreceive")
+test_dict_receive()
+
+println("Test completed.")

test/test_julia_dict_sender.jl

@@ -0,0 +1,137 @@
+#!/usr/bin/env julia
+# Test script for Dictionary transport testing
+# Tests sending 1 large and 1 small Dictionaries via direct and link transport
+# Uses NATSBridge.jl smartsend with "dictionary" type
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_dict_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+# Create correlation ID for tracing
+correlation_id = string(uuid4())
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test dictionary transfer                                  #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] [Correlation: $correlation_id] $message")
+end
+
+
+# File upload handler for plik server
+function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+    # Get upload ID
+    url_getUploadID = "$fileserver_url/upload"
+    headers = ["Content-Type" => "application/json"]
+    body = """{ "OneShot" : true }"""
+    httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false)
+    responseJson = JSON.parse(String(httpResponse.body))
+    uploadid = responseJson["id"]
+    uploadtoken = responseJson["uploadToken"]
+    
+    # Upload file
+    file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream")
+    url_upload = "$fileserver_url/file/$uploadid"
+    headers = ["X-UploadToken" => uploadtoken]
+    
+    form = HTTP.Form(Dict("file" => file_multipart))
+    httpResponse = HTTP.post(url_upload, headers, form)
+    responseJson = JSON.parse(String(httpResponse.body))
+    
+    fileid = responseJson["id"]
+    url = "$fileserver_url/file/$uploadid/$fileid/$dataname"
+    
+    return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url)
+end
+
+
+# Sender: Send Dictionaries via smartsend
+function test_dict_send()
+    # Create a small Dictionary (will use direct transport)
+    small_dict = Dict(
+        "name" => "Alice",
+        "age" => 30,
+        "scores" => [95, 88, 92],
+        "metadata" => Dict(
+            "height" => 155,
+            "weight" => 55
+        )
+    )
+    
+    # Create a large Dictionary (will use link transport if > 1MB)
+    # Generate a larger dataset (~2MB to ensure link transport)
+    large_dict = Dict(
+        "ids" => collect(1:50000),
+        "names" => ["User_$i" for i in 1:50000],
+        "scores" => rand(1:100, 50000),
+        "categories" => ["Category_$(rand(1:10))" for i in 1:50000],
+        "metadata" => Dict(
+            "source" => "test_generator",
+            "timestamp" => string(Dates.now())
+        )
+    )
+    
+    # Test data 1: small Dictionary
+    data1 = ("small_dict", small_dict, "dictionary")
+    
+    # Test data 2: large Dictionary
+    data2 = ("large_dict", large_dict, "dictionary")
+    
+    # Use smartsend with dictionary type
+    # For small Dictionary: will use direct transport (JSON encoded)
+    # For large Dictionary: will use link transport (uploaded to fileserver)
+    env, env_json_str = NATSBridge.smartsend(
+        SUBJECT,
+        [data1, data2];  # List of (dataname, data, type) tuples
+        broker_url = NATS_URL,
+        fileserver_url = FILESERVER_URL,
+        fileserver_upload_handler = plik_upload_handler,
+        size_threshold = 1_000_000,  # 1MB threshold
+        correlation_id = correlation_id,
+        msg_purpose = "chat",
+        sender_name = "dict_sender",
+        receiver_name = "",
+        receiver_id = "",
+        reply_to = "",
+        reply_to_msg_id = "",
+        is_publish = true  # Publish the message to NATS
+    )
+    
+    log_trace("Sent message with $(length(env.payloads)) payloads")
+    
+    # Log transport type for each payload
+    for (i, payload) in enumerate(env.payloads)
+        log_trace("Payload $i ('$payload.dataname'):")
+        log_trace("  Transport: $(payload.transport)")
+        log_trace("  Type: $(payload.payload_type)")
+        log_trace("  Size: $(payload.size) bytes")
+        log_trace("  Encoding: $(payload.encoding)")
+        
+        if payload.transport == "link"
+            log_trace("  URL: $(payload.data)")
+        end
+    end
+end
+
+
+# Run the test
+println("Starting Dictionary transport test...")
+println("Correlation ID: $correlation_id")
+
+# Run sender
+println("start smartsend for dictionaries")
+test_dict_send()
+
+println("Test completed.")

test/test_julia_file_receiver.jl

@@ -0,0 +1,84 @@
+#!/usr/bin/env julia
+# Test script for large payload testing using binary transport
+# Tests sending a large file (> 1MB) via smartsend with binary type
+# Updated to match NATSBridge.jl API
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+
+# workdir = 
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test file transfer                                        #
+# ------------------------------------------------------------------------------------------------ #
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+# Receiver: Listen for messages and verify large payload handling
+function test_large_binary_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Use NATSBridge.smartreceive to handle the data
+        # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay)
+        result = NATSBridge.smartreceive(
+            msg;
+            max_retries = 5,
+            base_delay = 100,
+            max_delay = 5000
+        )
+        
+        # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples
+        for (dataname, data, data_type) in result["payloads"]
+            # Check transport type from the envelope
+            # For link transport, data is the URL string
+            # For direct transport, data is the actual payload bytes
+            
+            if isa(data, Vector{UInt8})
+                file_size = length(data)
+                log_trace("Received $(file_size) bytes of binary data for '$dataname' of type $data_type")
+                
+                # Save received data to a test file
+                output_path = "./new_$dataname"
+                write(output_path, data)
+                log_trace("Saved received data to $output_path")
+            else
+                log_trace("Received $(file_size) bytes of binary data for '$dataname' of type $data_type")
+            end
+        end
+    end
+    
+    # Keep listening for 10 seconds
+    sleep(120)  
+    NATS.drain(conn)
+end
+
+
+# Run the test
+println("Starting large binary payload test...")
+
+# # Run sender first
+# println("start smartsend")
+# test_large_binary_send()
+
+# Run receiver
+println("testing smartreceive")
+test_large_binary_receive()
+
+println("Test completed.")

test/test_julia_file_sender.jl

@@ -0,0 +1,123 @@
+#!/usr/bin/env julia
+# Test script for large payload testing using binary transport
+# Tests sending a large file (> 1MB) via smartsend with binary type
+# Updated to match NATSBridge.jl API
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+
+# workdir = 
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+# Create correlation ID for tracing
+correlation_id = string(uuid4())
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test file transfer                                        #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] [Correlation: $correlation_id] $message")
+end
+
+# File upload handler for plik server
+function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+    # Get upload ID
+    url_getUploadID = "$fileserver_url/upload"
+    headers = ["Content-Type" => "application/json"]
+    body = """{ "OneShot" : true }"""
+    httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false)
+    responseJson = JSON.parse(String(httpResponse.body))
+    uploadid = responseJson["id"]
+    uploadtoken = responseJson["uploadToken"]
+    
+    # Upload file
+    file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream")
+    url_upload = "$fileserver_url/file/$uploadid"
+    headers = ["X-UploadToken" => uploadtoken]
+    
+    form = HTTP.Form(Dict("file" => file_multipart))
+    httpResponse = HTTP.post(url_upload, headers, form)
+    responseJson = JSON.parse(String(httpResponse.body))
+    
+    fileid = responseJson["id"]
+    url = "$fileserver_url/file/$uploadid/$fileid/$dataname"
+    
+    return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url)
+end
+
+# Sender: Send large binary file via smartsend
+function test_large_binary_send()
+    # Read the large file as binary data
+    
+    # test data 1
+    file_path1 = "./testFile_large.zip"
+    file_data1 = read(file_path1)
+    filename1 = basename(file_path1)
+    data1 = (filename1, file_data1, "binary")
+
+    # test data 2
+    file_path2 = "./testFile_small.zip"
+    file_data2 = read(file_path2)
+    filename2 = basename(file_path2)
+    data2 = (filename2, file_data2, "binary")
+    
+    
+    
+    # Use smartsend with binary type - will automatically use link transport
+    # if file size exceeds the threshold (1MB by default)
+    # API: smartsend(subject, [(dataname, data, type), ...]; keywords...)
+    env, env_json_str = NATSBridge.smartsend(
+        SUBJECT,
+        [data1, data2];  # List of (dataname, data, type) tuples
+        broker_url = NATS_URL;
+        fileserver_url = FILESERVER_URL,
+        fileserver_upload_handler = plik_upload_handler,
+        size_threshold = 1_000_000,
+        correlation_id = correlation_id,
+        msg_purpose = "chat",
+        sender_name = "sender",
+        receiver_name = "",
+        receiver_id = "",
+        reply_to = "",
+        reply_to_msg_id = "",
+        is_publish = true  # Publish the message to NATS
+    )
+    
+    log_trace("Sent message with transport: $(env.payloads[1].transport)")
+    log_trace("Envelope type: $(env.payloads[1].payload_type)")
+    
+    # Check if link transport was used
+    if env.payloads[1].transport == "link"
+        log_trace("Using link transport - file uploaded to HTTP server")
+        log_trace("URL: $(env.payloads[1].data)")
+    else
+        log_trace("Using direct transport - payload sent via NATS")
+    end
+end
+
+# Run the test
+println("Starting large binary payload test...")
+println("Correlation ID: $correlation_id")
+
+# Run sender first
+println("start smartsend")
+test_large_binary_send()
+
+# Run receiver
+# println("testing smartreceive")
+# test_large_binary_receive()
+
+println("Test completed.")

test/test_julia_mix_payloads_receiver.jl

@@ -0,0 +1,228 @@
+#!/usr/bin/env julia
+# Test script for mixed-content message testing
+# Tests receiving a mix of text, json, table, image, audio, video, and binary data
+# from Julia serviceA to Julia serviceB using NATSBridge.jl smartreceive
+#
+# This test demonstrates that any combination and any number of mixed content
+# can be sent and received correctly.
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_mix_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                      test mixed content transfer                                 #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+
+# Receiver: Listen for messages and verify mixed content handling
+function test_mix_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Use NATSBridge.smartreceive to handle the data
+        # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay)
+        result = NATSBridge.smartreceive(
+            msg;
+            max_retries = 5,
+            base_delay = 100,
+            max_delay = 5000
+        )
+        
+        log_trace("Received $(length(result["payloads"])) payloads")
+        
+        # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples
+        for (dataname, data, data_type) in result["payloads"]
+            log_trace("\n=== Payload: $dataname (type: $data_type) ===")
+            
+            # Handle different data types
+            if data_type == "text"
+                # Text data - should be a String
+                if isa(data, String)
+                    log_trace("  Type: String")
+                    log_trace("  Length: $(length(data)) characters")
+                    
+                    # Display first 200 characters
+                    if length(data) > 200
+                        log_trace("  First 200 chars: $(data[1:200])...")
+                    else
+                        log_trace("  Content: $data")
+                    end
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.txt"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected String, got $(typeof(data))")
+                end
+                
+            elseif data_type == "dictionary"
+                # Dictionary data - should be JSON object
+                if isa(data, JSON.Object{String, Any})
+                    log_trace("  Type: Dict")
+                    log_trace("  Keys: $(keys(data))")
+                    
+                    # Display nested content
+                    for (key, value) in data
+                        log_trace("  $key => $value")
+                    end
+                    
+                    # Save to JSON file
+                    output_path = "./received_$dataname.json"
+                    json_str = JSON.json(data, 2)
+                    write(output_path, json_str)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Dict, got $(typeof(data))")
+                end
+                
+            elseif data_type == "table"
+                # Table data - should be a DataFrame
+                data = DataFrame(data)
+                if isa(data, DataFrame)
+                    log_trace("  Type: DataFrame")
+                    log_trace("  Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns")
+                    log_trace("  Columns: $(names(data))")
+                    
+                    # Display first few rows
+                    log_trace("  First 5 rows:")
+                    display(data[1:min(5, size(data, 1)), :])
+                    
+                    # Save to Arrow file
+                    output_path = "./received_$dataname.arrow"
+                    io = IOBuffer()
+                    Arrow.write(io, data)
+                    write(output_path, take!(io))
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected DataFrame, got $(typeof(data))")
+                end
+                
+            elseif data_type == "image"
+                # Image data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "audio"
+                # Audio data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "video"
+                # Video data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "binary"
+                # Binary data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            else
+                log_trace("  ERROR: Unknown data type '$data_type'")
+            end
+        end
+        
+        # Summary
+        println("\n=== Verification Summary ===")
+        text_count = count(x -> x[3] == "text", result["payloads"])
+        dict_count = count(x -> x[3] == "dictionary", result["payloads"])
+        table_count = count(x -> x[3] == "table", result["payloads"])
+        image_count = count(x -> x[3] == "image", result["payloads"])
+        audio_count = count(x -> x[3] == "audio", result["payloads"])
+        video_count = count(x -> x[3] == "video", result["payloads"])
+        binary_count = count(x -> x[3] == "binary", result["payloads"])
+        
+        log_trace("Text payloads: $text_count")
+        log_trace("Dictionary payloads: $dict_count")
+        log_trace("Table payloads: $table_count")
+        log_trace("Image payloads: $image_count")
+        log_trace("Audio payloads: $audio_count")
+        log_trace("Video payloads: $video_count")
+        log_trace("Binary payloads: $binary_count")
+        
+        # Print transport type info for each payload if available
+        println("\n=== Payload Details ===")
+        for (dataname, data, data_type) in result["payloads"]
+            if data_type in ["image", "audio", "video", "binary"]
+                log_trace("$dataname: $(length(data)) bytes (binary)")
+            elseif data_type == "table"
+              data = DataFrame(data)
+                log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)")
+            elseif data_type == "dictionary"
+                log_trace("$dataname: $(length(JSON.json(data))) bytes (Dict)")
+            elseif data_type == "text"
+                log_trace("$dataname: $(length(data)) characters (String)")
+            end
+        end
+    end
+    
+    # Keep listening for 2 minutes
+    sleep(120)  
+    NATS.drain(conn)
+end
+
+
+# Run the test
+println("Starting mixed-content transport test...")
+println("Note: This receiver will wait for messages from the sender.")
+println("Run test_julia_to_julia_mix_sender.jl first to send test data.")
+
+# Run receiver
+println("\ntesting smartreceive for mixed content")
+test_mix_receive()
+
+println("\nTest completed.")

test/test_julia_mix_payloads_sender.jl

@@ -0,0 +1,239 @@
+#!/usr/bin/env julia
+# Test script for mixed-content message testing
+# Tests sending a mix of text, json, table, image, audio, video, and binary data
+# from Julia serviceA to Julia serviceB using NATSBridge.jl smartsend
+# 
+# This test demonstrates that any combination and any number of mixed content
+# can be sent and received correctly.
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_mix_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+# Create correlation ID for tracing
+correlation_id = string(uuid4())
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                      test mixed content transfer                                 #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] [Correlation: $correlation_id] $message")
+end
+
+
+# File upload handler for plik server
+function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+    # Get upload ID
+    url_getUploadID = "$fileserver_url/upload"
+    headers = ["Content-Type" => "application/json"]
+    body = """{ "OneShot" : true }"""
+    httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false)
+    responseJson = JSON.parse(String(httpResponse.body))
+    uploadid = responseJson["id"]
+    uploadtoken = responseJson["uploadToken"]
+    
+    # Upload file
+    file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream")
+    url_upload = "$fileserver_url/file/$uploadid"
+    headers = ["X-UploadToken" => uploadtoken]
+    
+    form = HTTP.Form(Dict("file" => file_multipart))
+    httpResponse = HTTP.post(url_upload, headers, form)
+    responseJson = JSON.parse(String(httpResponse.body))
+    
+    fileid = responseJson["id"]
+    url = "$fileserver_url/file/$uploadid/$fileid/$dataname"
+    
+    return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url)
+end
+
+
+# Helper: Create sample data for each type
+function create_sample_data()
+    # Text data (small - direct transport)
+    text_data = "Hello! This is a test chat message. 🎉\nHow are you doing today? 😊"
+    
+    # Dictionary/JSON data (medium - could be direct or link)
+    dict_data = Dict(
+        "type" => "chat",
+        "sender" => "serviceA",
+        "receiver" => "serviceB",
+        "metadata" => Dict(
+            "timestamp" => string(Dates.now()),
+            "priority" => "high",
+            "tags" => ["urgent", "chat", "test"]
+        ),
+        "content" => Dict(
+            "text" => "This is a JSON-formatted chat message with nested structure.",
+            "format" => "markdown",
+            "mentions" => ["user1", "user2"]
+        )
+    )
+    
+    # Table data (DataFrame - small - direct transport)
+    table_data_small = DataFrame(
+        id = 1:10,
+        message = ["msg_$i" for i in 1:10],
+        sender = ["sender_$i" for i in 1:10],
+        timestamp = [string(Dates.now()) for _ in 1:10],
+        priority = rand(1:3, 10)
+    )
+    
+    # Table data (DataFrame - large - link transport)
+    # ~1.5MB of data (150,000 rows) - should trigger link transport
+    table_data_large = DataFrame(
+        id = 1:150_000,
+        message = ["msg_$i" for i in 1:150_000],
+        sender = ["sender_$i" for i in 1:150_000],
+        timestamp = [string(Dates.now()) for i in 1:150_000],
+        priority = rand(1:3, 150_000)
+    )
+    
+    # Image data (small binary - direct transport)
+    # Create a simple 10x10 pixel PNG-like data (128 bytes header + 100 pixels = 112 bytes)
+    # Using simple RGB data (10*10*3 = 300 bytes of pixel data)
+    image_width = 10
+    image_height = 10
+    image_data = UInt8[]
+    # PNG header (simplified)
+    push!(image_data, 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A)
+    # Simple RGB data (RGBRGBRGB...)
+    for i in 1:image_width*image_height
+        push!(image_data, 0xFF, 0x00, 0x00)  # Red pixel
+    end
+    
+    # Image data (large - link transport)
+    # Create a larger image (~1.5MB) to test link transport
+    large_image_width = 500
+    large_image_height = 1000
+    large_image_data = UInt8[]
+    # PNG header (simplified for 500x1000)
+    push!(large_image_data, 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A)
+    # RGB data (500*1000*3 = 1,500,000 bytes)
+    for i in 1:large_image_width*large_image_height
+        push!(large_image_data, rand(1:255), rand(1:255), rand(1:255))  # Random color pixels
+    end
+    
+    # Audio data (small binary - direct transport)
+    audio_data = UInt8[rand(1:255) for _ in 1:100]
+    
+    # Audio data (large - link transport)
+    # ~1.5MB of audio-like data
+    large_audio_data = UInt8[rand(1:255) for _ in 1:1_500_000]
+    
+    # Video data (small binary - direct transport)
+    video_data = UInt8[rand(1:255) for _ in 1:150]
+    
+    # Video data (large - link transport)
+    # ~1.5MB of video-like data
+    large_video_data = UInt8[rand(1:255) for _ in 1:1_500_000]
+    
+    # Binary data (small - direct transport)
+    binary_data = UInt8[rand(1:255) for _ in 1:200]
+    
+    # Binary data (large - link transport)
+    # ~1.5MB of binary data
+    large_binary_data = UInt8[rand(1:255) for _ in 1:1_500_000]
+    
+    return (
+        text_data,
+        dict_data,
+        table_data_small,
+        table_data_large,
+        image_data,
+        large_image_data,
+        audio_data,
+        large_audio_data,
+        video_data,
+        large_video_data,
+        binary_data,
+        large_binary_data
+    )
+end
+
+
+# Sender: Send mixed content via smartsend
+function test_mix_send()
+    # Create sample data
+    (text_data, dict_data, table_data_small, table_data_large, image_data, large_image_data, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data) = create_sample_data()
+    
+    # Create payloads list - mixed content with both small and large data
+    # Small data uses direct transport, large data uses link transport
+    payloads = [
+        # Small data (direct transport) - text, dictionary, small table
+        ("chat_text", text_data, "text"),
+        ("chat_json", dict_data, "dictionary"),
+        ("chat_table_small", table_data_small, "table"),
+        
+        # Large data (link transport) - large table, large image, large audio, large video, large binary
+        ("chat_table_large", table_data_large, "table"),
+        ("user_image_large", large_image_data, "image"),
+        ("audio_clip_large", large_audio_data, "audio"),
+        ("video_clip_large", large_video_data, "video"),
+        ("binary_file_large", large_binary_data, "binary")
+    ]
+    
+    # Use smartsend with mixed content
+    env, env_json_str = NATSBridge.smartsend(
+        SUBJECT,
+        payloads;  # List of (dataname, data, type) tuples
+        broker_url = NATS_URL,
+        fileserver_url = FILESERVER_URL,
+        fileserver_upload_handler = plik_upload_handler,
+        size_threshold = 1_000_000,  # 1MB threshold
+        correlation_id = correlation_id,
+        msg_purpose = "chat",
+        sender_name = "mix_sender",
+        receiver_name = "",
+        receiver_id = "",
+        reply_to = "",
+        reply_to_msg_id = "",
+        is_publish = true  # Publish the message to NATS
+    )
+    
+    log_trace("Sent message with $(length(env.payloads)) payloads")
+    
+    # Log transport type for each payload
+    for (i, payload) in enumerate(env.payloads)
+        log_trace("Payload $i ('$payload.dataname'):")
+        log_trace("  Transport: $(payload.transport)")
+        log_trace("  Type: $(payload.payload_type)")
+        log_trace("  Size: $(payload.size) bytes")
+        log_trace("  Encoding: $(payload.encoding)")
+        
+        if payload.transport == "link"
+            log_trace("  URL: $(payload.data)")
+        end
+    end
+    
+    # Summary
+    println("\n--- Transport Summary ---")
+    direct_count = count(p -> p.transport == "direct", env.payloads)
+    link_count = count(p -> p.transport == "link", env.payloads)
+    log_trace("Direct transport: $direct_count payloads")
+    log_trace("Link transport: $link_count payloads")
+end
+
+
+# Run the test
+println("Starting mixed-content transport test...")
+println("Correlation ID: $correlation_id")
+
+# Run sender
+println("start smartsend for mixed content")
+test_mix_send()
+
+println("\nTest completed.")
+println("Note: Run test_julia_to_julia_mix_receiver.jl to receive the messages.")

test/test_julia_table_receiver.jl

@@ -0,0 +1,84 @@
+#!/usr/bin/env julia
+# Test script for DataFrame table transport testing
+# Tests receiving 1 large and 1 small DataFrames via direct and link transport
+# Uses NATSBridge.jl smartreceive with "table" type
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_table_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test table transfer                                       #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+
+# Receiver: Listen for messages and verify DataFrame table handling
+function test_table_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Use NATSBridge.smartreceive to handle the data
+        # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay)
+        result = NATSBridge.smartreceive(
+            msg;
+            max_retries = 5,
+            base_delay = 100,
+            max_delay = 5000
+        )
+        
+        # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples
+        for (dataname, data, data_type) in result["payloads"]
+          data = DataFrame(data)
+            if isa(data, DataFrame)
+                log_trace("Received DataFrame '$dataname' of type $data_type")
+                log_trace("  Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns")
+                log_trace("  Column names: $(names(data))")
+                
+                # Display first few rows
+                println("  First 5 rows:")
+                display(data[1:min(5, size(data, 1)), :])
+                
+                # Save to file
+                output_path = "./received_$dataname.arrow"
+                io = IOBuffer()
+                Arrow.write(io, data)
+                write(output_path, take!(io))
+                log_trace("Saved DataFrame to $output_path")
+            else
+                log_trace("Received unexpected data type for '$dataname': $(typeof(data))")
+            end
+        end
+    end
+    
+    # Keep listening for 10 seconds
+    sleep(120)  
+    NATS.drain(conn)
+end
+
+
+# Run the test
+println("Starting DataFrame table transport test...")
+println("Note: This receiver will wait for messages from the sender.")
+println("Run test_julia_to_julia_table_sender.jl first to send test data.")
+
+# Run receiver
+println("testing smartreceive")
+test_table_receive()
+
+println("Test completed.")

test/test_julia_table_sender.jl

@@ -0,0 +1,135 @@
+#!/usr/bin/env julia
+# Test script for DataFrame table transport testing
+# Tests sending 1 large and 1 small DataFrames via direct and link transport
+# Uses NATSBridge.jl smartsend with "table" type
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_table_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+# Create correlation ID for tracing
+correlation_id = string(uuid4())
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test table transfer                                       #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] [Correlation: $correlation_id] $message")
+end
+
+
+# File upload handler for plik server
+function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+    # Get upload ID
+    url_getUploadID = "$fileserver_url/upload"
+    headers = ["Content-Type" => "application/json"]
+    body = """{ "OneShot" : true }"""
+    httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false)
+    responseJson = JSON.parse(String(httpResponse.body))
+    uploadid = responseJson["id"]
+    uploadtoken = responseJson["uploadToken"]
+    
+    # Upload file
+    file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream")
+    url_upload = "$fileserver_url/file/$uploadid"
+    headers = ["X-UploadToken" => uploadtoken]
+    
+    form = HTTP.Form(Dict("file" => file_multipart))
+    httpResponse = HTTP.post(url_upload, headers, form)
+    responseJson = JSON.parse(String(httpResponse.body))
+    
+    fileid = responseJson["id"]
+    url = "$fileserver_url/file/$uploadid/$fileid/$dataname"
+    
+    return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url)
+end
+
+
+# Sender: Send DataFrame tables via smartsend
+function test_table_send()
+    # Create a small DataFrame (will use direct transport)
+    small_df = DataFrame(
+        id = 1:10,
+        name = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"],
+        score = [95, 88, 92, 85, 90, 78, 95, 88, 92, 85],
+        category = ["A", "B", "A", "B", "A", "B", "A", "B", "A", "B"]
+    )
+    
+    # Create a large DataFrame (will use link transport if > 1MB)
+    # Generate a larger dataset (~2MB to ensure link transport)
+    large_ids = 1:50000
+    large_names = ["User_$i" for i in 1:50000]
+    large_scores = rand(1:100, 50000)
+    large_categories = ["Category_$(rand(1:10))" for i in 1:50000]
+    
+    large_df = DataFrame(
+        id = large_ids,
+        name = large_names,
+        score = large_scores,
+        category = large_categories
+    )
+    
+    # Test data 1: small DataFrame
+    data1 = ("small_table", small_df, "table")
+    
+    # Test data 2: large DataFrame
+    data2 = ("large_table", large_df, "table")
+    
+    # Use smartsend with table type
+    # For small DataFrame: will use direct transport (Base64 encoded Arrow IPC)
+    # For large DataFrame: will use link transport (uploaded to fileserver)
+    env, env_json_str = NATSBridge.smartsend(
+        SUBJECT,
+        [data1, data2];  # List of (dataname, data, type) tuples
+        broker_url = NATS_URL,
+        fileserver_url = FILESERVER_URL,
+        fileserver_upload_handler = plik_upload_handler,
+        size_threshold = 1_000_000,  # 1MB threshold
+        correlation_id = correlation_id,
+        msg_purpose = "chat",
+        sender_name = "table_sender",
+        receiver_name = "",
+        receiver_id = "",
+        reply_to = "",
+        reply_to_msg_id = "",
+        is_publish = true  # Publish the message to NATS
+    )
+    
+    log_trace("Sent message with $(length(env.payloads)) payloads")
+    
+    # Log transport type for each payload
+    for (i, payload) in enumerate(env.payloads)
+        log_trace("Payload $i ('$payload.dataname'):")
+        log_trace("  Transport: $(payload.transport)")
+        log_trace("  Type: $(payload.payload_type)")
+        log_trace("  Size: $(payload.size) bytes")
+        log_trace("  Encoding: $(payload.encoding)")
+        
+        if payload.transport == "link"
+            log_trace("  URL: $(payload.data)")
+        end
+    end
+end
+
+
+# Run the test
+println("Starting DataFrame table transport test...")
+println("Correlation ID: $correlation_id")
+
+# Run sender
+println("start smartsend for tables")
+test_table_send()
+
+println("Test completed.")

test/test_julia_text_receiver.jl

@@ -0,0 +1,83 @@
+#!/usr/bin/env julia
+# Test script for text transport testing
+# Tests receiving 1 large and 1 small text from Julia serviceA to Julia serviceB
+# Uses NATSBridge.jl smartreceive with "text" type
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_text_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test text transfer                                        #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+
+# Receiver: Listen for messages and verify text handling
+function test_text_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Use NATSBridge.smartreceive to handle the data
+        # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay)
+        result = NATSBridge.smartreceive(
+            msg;
+            max_retries = 5,
+            base_delay = 100,
+            max_delay = 5000
+        )
+        
+        # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples
+        for (dataname, data, data_type) in result["payloads"]
+            if isa(data, String)
+                log_trace("Received text '$dataname' of type $data_type")
+                log_trace("  Length: $(length(data)) characters")
+                
+                # Display first 100 characters
+                if length(data) > 100
+                    log_trace("  First 100 characters: $(data[1:100])...")
+                else
+                    log_trace("  Content: $data")
+                end
+                
+                # Save to file
+                output_path = "./received_$dataname.txt"
+                write(output_path, data)
+                log_trace("Saved text to $output_path")
+            else
+                log_trace("Received unexpected data type for '$dataname': $(typeof(data))")
+            end
+        end
+    end
+    
+    # Keep listening for 10 seconds
+    sleep(120)  
+    NATS.drain(conn)
+end
+
+
+# Run the test
+println("Starting text transport test...")
+println("Note: This receiver will wait for messages from the sender.")
+println("Run test_julia_to_julia_text_sender.jl first to send test data.")
+
+# Run receiver
+println("testing smartreceive for text")
+test_text_receive()
+
+println("Test completed.")

test/test_julia_text_sender.jl

@@ -0,0 +1,120 @@
+#!/usr/bin/env julia
+# Test script for text transport testing
+# Tests sending 1 large and 1 small text from Julia serviceA to Julia serviceB
+# Uses NATSBridge.jl smartsend with "text" type
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP
+
+# Include the bridge module
+include("../src/NATSBridge.jl")
+using .NATSBridge
+
+# Configuration
+const SUBJECT = "/NATSBridge_text_test"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+# Create correlation ID for tracing
+correlation_id = string(uuid4())
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                        test text transfer                                        #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] [Correlation: $correlation_id] $message")
+end
+
+
+# File upload handler for plik server
+function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+    # Get upload ID
+    url_getUploadID = "$fileserver_url/upload"
+    headers = ["Content-Type" => "application/json"]
+    body = """{ "OneShot" : true }"""
+    httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false)
+    responseJson = JSON.parse(String(httpResponse.body))
+    uploadid = responseJson["id"]
+    uploadtoken = responseJson["uploadToken"]
+    
+    # Upload file
+    file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream")
+    url_upload = "$fileserver_url/file/$uploadid"
+    headers = ["X-UploadToken" => uploadtoken]
+    
+    form = HTTP.Form(Dict("file" => file_multipart))
+    httpResponse = HTTP.post(url_upload, headers, form)
+    responseJson = JSON.parse(String(httpResponse.body))
+    
+    fileid = responseJson["id"]
+    url = "$fileserver_url/file/$uploadid/$fileid/$dataname"
+    
+    return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url)
+end
+
+
+# Sender: Send text via smartsend
+function test_text_send()
+    # Create a small text (will use direct transport)
+    small_text = "Hello, this is a small text message. Testing direct transport via NATS."
+    
+    # Create a large text (will use link transport if > 1MB)
+    # Generate a larger text (~2MB to ensure link transport)
+    large_text = join(["Line $i: This is a sample text line with some content to pad the size. " for i in 1:50000], "")
+    
+    # Test data 1: small text
+    data1 = ("small_text", small_text, "text")
+    
+    # Test data 2: large text
+    data2 = ("large_text", large_text, "text")
+    
+    # Use smartsend with text type
+    # For small text: will use direct transport (Base64 encoded UTF-8)
+    # For large text: will use link transport (uploaded to fileserver)
+    env, env_json_str = NATSBridge.smartsend(
+        SUBJECT,
+        [data1, data2];  # List of (dataname, data, type) tuples
+        broker_url = NATS_URL,
+        fileserver_url = FILESERVER_URL,
+        fileserver_upload_handler = plik_upload_handler,
+        size_threshold = 1_000_000,  # 1MB threshold
+        correlation_id = correlation_id,
+        msg_purpose = "chat",
+        sender_name = "text_sender",
+        receiver_name = "",
+        receiver_id = "",
+        reply_to = "",
+        reply_to_msg_id = "",
+        is_publish = true  # Publish the message to NATS
+    )
+    
+    log_trace("Sent message with $(length(env.payloads)) payloads")
+    
+    # Log transport type for each payload
+    for (i, payload) in enumerate(env.payloads)
+        log_trace("Payload $i ('$payload.dataname'):")
+        log_trace("  Transport: $(payload.transport)")
+        log_trace("  Type: $(payload.payload_type)")
+        log_trace("  Size: $(payload.size) bytes")
+        log_trace("  Encoding: $(payload.encoding)")
+        
+        if payload.transport == "link"
+            log_trace("  URL: $(payload.data)")
+        end
+    end
+end
+
+
+# Run the test
+println("Starting text transport test...")
+println("Correlation ID: $correlation_id")
+
+# Run sender
+println("start smartsend for text")
+test_text_send()
+
+println("Test completed.")

test/test_large_payload.jl

@@ -1,139 +0,0 @@
-#!/usr/bin/env julia
-# Test script for large payload testing using binary transport
-# Tests sending a large file (> 1MB) via smartsend with binary type
-
-using NATS, JSON, UUIDs, Dates
-
-# Include the bridge module
-include("../src/NATSBridge.jl")
-using .NATSBridge
-
-# Configuration
-const SUBJECT = "/large_binary_test"
-const NATS_URL = "nats.yiem.cc"
-const FILESERVER_URL = "http://192.168.88.104:8080"
-
-# Create correlation ID for tracing
-correlation_id = string(uuid4())
-
-# File path for large binary payload test
-const LARGE_FILE_PATH = "./testFile_small.zip"
-const filename = basename(LARGE_FILE_PATH)
-
-# Helper: Log with correlation ID
-function log_trace(message)
-    timestamp = Dates.now()
-    println("[$timestamp] [Correlation: $correlation_id] $message")
-end
-
-# Sender: Send large binary file via smartsend
-function test_large_binary_send()
-    conn = NATS.connect(NATS_URL)
-    # Read the large file as binary data
-    log_trace("Reading large file: $LARGE_FILE_PATH")
-    file_data = read(LARGE_FILE_PATH)
-    
-    file_size = length(file_data)
-    log_trace("File size: $file_size bytes")
-    
-    # Use smartsend with binary type - will automatically use link transport
-    # if file size exceeds the threshold (1MB by default)
-    env = NATSBridge.smartsend(
-        SUBJECT,
-        file_data,
-        "binary",
-        nats_url = NATS_URL,
-        fileserver_url = FILESERVER_URL;
-        dataname=filename
-    )
-    
-    log_trace("Sent message with transport: $(env.transport)")
-    log_trace("Envelope type: $(env.type)")
-    
-    # Check if link transport was used
-    if env.transport == "link"
-        log_trace("Using link transport - file uploaded to HTTP server")
-        log_trace("URL: $(env.url)")
-    else
-        log_trace("Using direct transport - payload sent via NATS")
-    end
-    
-    NATS.drain(conn)
-end
-
-# Receiver: Listen for messages and verify large payload handling
-function test_large_binary_receive()
-  conn = NATS.connect(NATS_URL)
-  NATS.subscribe(conn, SUBJECT) do msg
-    log_trace("Received message on $(msg.subject)")
-    
-    # Use NATSBridge.smartreceive to handle the data
-    result = NATSBridge.smartreceive(msg)
-    # println("envelope----- ", result.envelope)
-    # Check transport type
-    if result.envelope.transport == "direct"
-        log_trace("Received direct transport with ---- bytes")
-    else
-        # For link transport, result.data is the URL
-        log_trace("Received link transport at ---")
-    end
-              
-    # Verify the received data matches the original
-    if result.envelope.type == "binary"
-      if isa(result.data, Vector{UInt8})
-        file_size = length(result.data)
-        log_trace("Received $(file_size) bytes of binary data")
-        
-        # Save received data to a test file
-        #[WORKING] add dataname so I know it is a file
-        filename = basename(result.envelope.url)
-        output_path = "./new_$filename"
-        write(output_path, result.data)
-        log_trace("Saved received data to $output_path")
-        
-        # Verify file size
-        original_size = length(read(LARGE_FILE_PATH))
-        if file_size == original_size
-            log_trace("SUCCESS: File size matches! Original: $original_size bytes")
-        else
-            log_trace("WARNING: File size mismatch! Original: $original_size, Received: $file_size")
-        end
-      end
-    end
-  end
-
-  # Keep listening for 10 seconds
-  sleep(60)  
-  NATS.drain(conn)
-end
-
-# Run the test
-println("Starting large binary payload test...")
-println("Correlation ID: $correlation_id")
-println("Large file: $LARGE_FILE_PATH")
-
-# # Run sender first
-# println("start smartsend")
-# test_large_binary_send()
-
-# Run receiver
-println("testing smartreceive")
-test_large_binary_receive()
-
-println("Test completed.")
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-