diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e710a67 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +package.json +package-lock.json diff --git a/AI_prompt.txt b/AI_prompt.txt index 5120027..2df5aab 100644 --- a/AI_prompt.txt +++ b/AI_prompt.txt @@ -18,12 +18,9 @@ Create a walkthrough for Julia service-A service sending a mix-content chat mess - I updated the following: - NATSBridge.jl. Essentially I add NATS_connection keyword and new publish_message function to support the keyword. - Use them and ONLY them as ground truth. - Then update the following files accordingly: - architecture.md - implementation.md @@ -39,11 +36,8 @@ All API should be semantically consistent and naming should be consistent across Task: Update NATSBridge.js to reflect recent changes in NATSBridge.jl and docs - Context: NATSBridge.jl and docs has been updated. - Requirements: - Source of Truth: Treat the updated NATSBridge.jl and docs as the definitive source. API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. @@ -51,3 +45,59 @@ Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read( + + + +I'm expanding this Julia package (NATSBridge) into a cross-platform project by adding a JavaScript and Python/MicroPython implementation. To ensure accuracy, the Julia src directory will serve as the ground truth, as the documentation may be outdated. + +My goal is to maintain interface parity at the high-level API for a consistent user experience, while ensuring the low-level implementation adheres strictly to the idiomatic conventions of each respective language (e.g., multiple dispatch in Julia vs. asynchronous, prototype, or class-based patterns in JS and Python/MicroPython) + +Now, help me do the following: +1) check architecture.md for any mistake. + + + + +Help me expands this Julia package (NATSBridge) into a cross-platform project by adding a JavaScript and Python/MicroPython implementation. To ensure accuracy, NATSBridge.jl will serve as the ground truth, as the documentation may be outdated. + +My goal is to maintain interface parity at the high-level API for a consistent user experience, while ensuring the low-level implementation adheres strictly to the idiomatic conventions of each respective language (e.g., multiple dispatch in Julia vs. asynchronous, prototype, or class-based patterns in JS and Python/MicroPython) + +Now do the following: +1) check docs to see if there is any mistake. + + + + + +I'm expanding this Julia package (NATSBridge) into a cross-platform project by adding +a JavaScript, Python and MicroPython implementation. +The following will serve as the ground truth: +- test_julia_mix_payloads_sender.jl +- NATSBridge.jl +- test_julia_mix_payloads_receiver.jl +- architecture.md + +My goal is to maintain interface parity at the high-level API for a consistent user experience, +while ensuring the low-level implementation adheres strictly to the idiomatic conventions of each +respective language (e.g., multiple dispatch in Julia vs. asynchronous, prototype, or class-based +patterns in JS, Python and MicroPython) + +Now, help me do the following: +1) Check whether natsbridge.js needs update or it already up to date. + + + + + + + + + + + + + + + + + diff --git a/README.md b/README.md index be3f1ce..1fdf508 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# NATSBridge +# NATSBridge - Cross-Platform Bi-Directional Data Bridge -A high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +A high-performance, bi-directional data bridge for **Julia, JavaScript, Python, and MicroPython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![NATS](https://img.shields.io/badge/NATS-Enabled-green.svg)](https://nats.io) @@ -10,22 +10,21 @@ A high-performance, bi-directional data bridge for **Julia** applications using ## Table of Contents - [Overview](#overview) +- [Cross-Platform Support](#cross-platform-support) - [Features](#features) -- [Architecture](#architecture) -- [Installation](#installation) - [Quick Start](#quick-start) - [API Reference](#api-reference) - [Payload Types](#payload-types) -- [Transport Strategies](#transport-strategies) -- [Examples](#examples) +- [Cross-Platform Examples](#cross-platform-examples) - [Testing](#testing) +- [Documentation](#documentation) - [License](#license) --- ## Overview -NATSBridge enables seamless communication for Julia applications through NATS, with intelligent transport selection based on payload size: +NATSBridge enables seamless communication across multiple platforms through NATS, with intelligent transport selection based on payload size: | Transport | Payload Size | Method | |-----------|--------------|--------| @@ -36,14 +35,39 @@ NATSBridge enables seamless communication for Julia applications through NATS, w - **Chat Applications**: Text, images, audio, video in a single message - **File Transfer**: Efficient transfer of large files using claim-check pattern -- **Streaming Data**: Sensor data, telemetry, and analytics pipelines +- **IoT/Embedded**: Sensor data, telemetry, and analytics pipelines (MicroPython) +- **Cross-Platform Communication**: Interoperability between Julia, JavaScript, Python, and MicroPython systems +--- + +## Cross-Platform Support + +| Platform | Implementation | Features | +|----------|----------------|----------| +| **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch | +| **JavaScript** | [`src/natsbridge.js`](src/natsbridge.js) | Node.js & browser, async/await | +| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints | +| **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API | + +### Platform Comparison + +| Feature | Julia | JavaScript | Python | MicroPython | +|---------|-------|------------|--------|-------------| +| Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | +| Async/Await | ❌ | ✅ Native | ✅ Native | ⚠️ (uasyncio) | +| Type Safety | ✅ Strong | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ | +| Arrow IPC | ✅ Native | ✅ | ✅ | ❌ | +| Direct Transport | ✅ | ✅ | ✅ | ✅ | +| Link Transport | ✅ | ✅ | ✅ | ⚠️ (Limited) | +| Handler Functions | ✅ | ✅ | ✅ | ✅ | +| Cross-Platform API | ✅ | ✅ | ✅ | ✅ | --- ## Features -- ✅ **Bi-directional messaging** for Julia applications +- ✅ **Cross-platform messaging** for Julia, JavaScript, Python, and MicroPython applications +- ✅ **Bi-directional messaging** with request-reply patterns - ✅ **Multi-payload support** - send multiple payloads with different types in one message - ✅ **Automatic transport selection** - direct vs link based on payload size - ✅ **Claim-Check pattern** for payloads > 1MB @@ -51,100 +75,7 @@ NATSBridge enables seamless communication for Julia applications through NATS, w - ✅ **Exponential backoff** for reliable file server downloads - ✅ **Correlation ID tracking** for message tracing - ✅ **Reply-to support** for request-response patterns -- ✅ **JetStream support** for message replay and durability - - ---- - -## Architecture - -### System Components - -```mermaid -flowchart TB - subgraph Sender["Julia Application (Sender)"] - SenderApp[App Code] - NATSBridge_Send[NATSBridge] - NATS_Client[NATS.jl] - end - - subgraph Receiver["Julia Application (Receiver)"] - ReceiverApp[App Code] - NATSBridge_Recv[NATSBridge] - NATS_Client_Recv[NATS.jl] - end - - subgraph Infrastructure["Infrastructure"] - NATS[NATS Server
Message Broker] - FileServer[HTTP File Server
Upload/Download] - end - - SenderApp --> NATSBridge_Send - NATSBridge_Send --> NATS_Client - NATS_Client --> NATS - - NATS --> NATS_Client_Recv - NATS_Client_Recv --> NATSBridge_Recv - NATSBridge_Recv --> ReceiverApp - - NATSBridge_Send -.->|HTTP POST upload| FileServer - FileServer -.->|HTTP GET download| NATSBridge_Recv - - style SenderApp fill:#e8f5e9 - style ReceiverApp fill:#e8f5e9 - style NATS fill:#fff3e0 - style FileServer fill:#f3e5f5 -``` - -### Key Components - -| Component | Description | -|-----------|-------------| -| **Julia Application** | Sender and receiver applications using the NATSBridge module | -| **NATS Server** | Message broker for transporting message envelopes | -| **HTTP File Server** | Independent HTTP server for large payload storage (e.g., Plik) | - -### Message Flow - -1. **Sender** creates a message envelope with payloads using `smartsend()` -2. **NATSBridge** serializes and encodes each payload based on type -3. **Transport Decision**: - - **Direct** (< 1MB): Payload encoded as Base64, published to NATS - - **Link** (≥ 1MB): Payload uploaded to HTTP file server, URL published to NATS -4. **NATS** routes message envelope to subscribers -5. **Receiver** receives message via NATS subscription callback -6. **NATSBridge** processes envelope: - - Decodes Base64 payloads from NATS message - - Fetches URLs from file server with exponential backoff -7. **Receiver** deserializes payloads based on their type - -### File Server Handler Abstraction - -The system uses handler functions to abstract file server operations: - -| Handler | Purpose | -|---------|---------| -| `plik_oneshot_upload()` | Uploads payload bytes to file server, returns URL | -| `_fetch_with_backoff()` | Downloads data from URL with exponential backoff retry | - -This abstraction allows support for different file server implementations (Plik, AWS S3, custom HTTP server). - ---- - -## Installation - -### Prerequisites - -- **NATS Server** (v2.10+ recommended) -- **HTTP File Server** (optional, for payloads > 1MB) - -### Julia - -```julia -using Pkg -Pkg.add("NATS") -Pkg.add("https://git.yiem.cc/ton/NATSBridge") -``` +- ✅ **Handler function abstraction** - pluggable file server implementations (Plik, AWS S3, custom) --- @@ -173,54 +104,72 @@ python3 -m http.server 8080 --directory /tmp/fileserver ```julia using NATSBridge -# Send a text message data = [("message", "Hello World", "text")] -env, env_json_str = NATSBridge.smartsend("/chat/room1", data; broker_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") println("Message sent!") ``` -### Step 4: Receive Messages +#### JavaScript -#### Julia +```javascript +const NATSBridge = require('./src/natsbridge.js'); -```julia -using NATS, NATSBridge +const data = [["message", "Hello World", "text"]]; +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat/room1", + data, + { broker_url: "nats://localhost:4222" } +); +console.log("Message sent!"); +``` -# Configuration -const SUBJECT = "/chat/room1" -const NATS_URL = "nats://localhost:4222" +#### Python -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] $message") -end +```python +from natsbridge import smartsend -# Receiver: Listen for messages - msg comes from the callback -function test_receive() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - log_trace("Received message on $(msg.subject)") - - # Receive and process message - env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) - for (dataname, data, type) in env["payloads"] - println("Received $dataname: $data") - end - end - - # Keep listening for 120 seconds - sleep(120) - NATS.drain(conn) -end - -test_receive() +data = [("message", "Hello World", "text")] +env, env_json_str = await smartsend( + "/chat/room1", + data, + broker_url="nats://localhost:4222" +) +print("Message sent!") ``` --- ## API Reference +### Unified API Standard + +All platforms use the same input/output format for payloads: + +**Input format for smartsend:** +``` +[(dataname1, data1, type1), (dataname2, data2, type2), ...] +``` + +**Output format for smartreceive:** +``` +{ + "correlation_id": "...", + "msg_id": "...", + "timestamp": "...", + "send_to": "...", + "msg_purpose": "...", + "sender_name": "...", + "sender_id": "...", + "receiver_name": "...", + "receiver_id": "...", + "reply_to": "...", + "reply_to_msg_id": "...", + "broker_url": "...", + "metadata": {...}, + "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] +} +``` + ### smartsend Sends data either directly via NATS or via a fileserver URL, depending on payload size. @@ -231,27 +180,96 @@ Sends data either directly via NATS or via a fileserver URL, depending on payloa using NATSBridge env, env_json_str = NATSBridge.smartsend( - subject, # NATS subject - data::AbstractArray{Tuple{String, Any, String}}; # List of (dataname, data, type) + subject::String, + data::AbstractArray{Tuple{String, Any, String}}; broker_url::String = "nats://localhost:4222", fileserver_url = "http://localhost:8080", fileserver_upload_handler::Function = plik_oneshot_upload, size_threshold::Int = 1_000_000, - correlation_id::String = string(uuid4()), # Correlation ID for tracing (auto-generated UUID) + correlation_id::String = string(uuid4()), msg_purpose::String = "chat", sender_name::String = "NATSBridge", receiver_name::String = "", receiver_id::String = "", reply_to::String = "", reply_to_msg_id::String = "", - is_publish::Bool = true, # Whether to automatically publish to NATS - NATS_connection::Union{NATS.Connection, Nothing} = nothing, # Pre-existing NATS connection (optional, saves connection overhead) - msg_id::String = string(uuid4()), # Message ID (auto-generated UUID) - sender_id::String = string(uuid4()) # Sender ID (auto-generated UUID) + is_publish::Bool = true, + NATS_connection::Union{NATS.Connection, Nothing} = nothing, + msg_id::String = string(uuid4()), + sender_id::String = string(uuid4()) ) # Returns: ::Tuple{msg_envelope_v1, String} -# - env: msg_envelope_v1 object with all envelope metadata and payloads -# - env_json_str: JSON string representation of the envelope for publishing +``` + +#### JavaScript + +```javascript +const NATSBridge = require('natsbridge'); + +const [env, env_json_str] = await NATSBridge.smartsend( + subject, + data, // Array of [dataname, data, type] tuples + { + broker_url: 'nats://localhost:4222', + fileserver_url: 'http://localhost:8080', + fileserver_upload_handler: NATSBridge.plikOneshotUpload, + size_threshold: 1_000_000, + correlation_id: uuidv4(), + msg_purpose: 'chat', + sender_name: 'NATSBridge', + receiver_name: '', + receiver_id: '', + reply_to: '', + reply_to_msg_id: '', + is_publish: true, + nats_connection: null, + msg_id: uuidv4(), + sender_id: uuidv4() + } +); +// Returns: Promise<[env, env_json_str]> +``` + +#### Python + +```python +from natsbridge import NATSBridge + +env, env_json_str = await NATSBridge.smartsend( + subject: str, + data: List[Tuple[str, Any, str]], + broker_url: str = "nats://localhost:4222", + fileserver_url: str = "http://localhost:8080", + fileserver_upload_handler: Callable = plik_oneshot_upload, + size_threshold: int = 1_000_000, + correlation_id: str = None, + msg_purpose: str = "chat", + sender_name: str = "NATSBridge", + receiver_name: str = "", + receiver_id: str = "", + reply_to: str = "", + reply_to_msg_id: str = "", + is_publish: bool = True, + nats_connection: Any = None, + msg_id: str = None, + sender_id: str = None +) +# Returns: Tuple[Dict, str] +``` + +#### MicroPython + +```python +from natsbridge import NATSBridge + +# Limited to direct transport (< 100KB threshold) +env, env_json_str = NATSBridge.smartsend( + subject, + data, # List of (dataname, data, type) tuples + broker_url="nats://localhost:4222", + size_threshold=100000 # Lower threshold for memory constraints +) +# Returns: Tuple[Dict, str] ``` ### smartreceive @@ -263,7 +281,6 @@ Receives and processes messages from NATS, handling both direct and link transpo ```julia using NATSBridge -# Note: msg is a NATS.Msg object passed from the subscription callback env = NATSBridge.smartreceive( msg::NATS.Msg; fileserver_download_handler::Function = _fetch_with_backoff, @@ -271,85 +288,75 @@ env = NATSBridge.smartreceive( base_delay::Int = 100, max_delay::Int = 5000 ) -# Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 +# Returns: ::JSON.Object{String, Any} ``` -### publish_message +#### JavaScript -Publish a message to a NATS subject. This function is available in Julia with two overloads: +```javascript +const env = await NATSBridge.smartreceive( + msg, + { + fileserver_download_handler: NATSBridge.fetchWithBackoff, + max_retries: 5, + base_delay: 100, + max_delay: 5000 + } +); +// Returns: Promise +``` -#### Julia +#### Python -**Using broker URL (creates new connection):** -```julia -using NATSBridge, NATS - -# Publish with URL - creates a new connection -NATSBridge.publish_message( - "nats://localhost:4222", # broker_url - "/chat/room1", # subject - "{\"correlation_id\":\"abc123\"}", # message - "abc123" # correlation_id +```python +env = await NATSBridge.smartreceive( + msg, + fileserver_download_handler=fetch_with_backoff, + max_retries=5, + base_delay=100, + max_delay=5000 ) +# Returns: Dict with "payloads" key ``` -**Using pre-existing connection (saves connection overhead):** -```julia -using NATSBridge, NATS +#### MicroPython -# Create connection once and reuse -conn = NATS.connect("nats://localhost:4222") -NATSBridge.publish_message(conn, "/chat/room1", "{\"correlation_id\":\"abc123\"}", "abc123") -# Connection is automatically drained after publish +```python +env = NATSBridge.smartreceive( + msg, + fileserver_download_handler=_sync_fileserver_download, + max_retries=3, + base_delay=100, + max_delay=1000 +) +# Returns: Dict with "payloads" key ``` --- ## Payload Types -| Type | Description | Serialization | -|------|-------------|---------------| -| `text` | Plain text strings | UTF-8 bytes | -| `dictionary` | JSON-serializable dictionaries | JSON | -| `table` | Tabular data (DataFrames, arrays) | Apache Arrow IPC | -| `image` | Image data (PNG, JPG) | Raw bytes | -| `audio` | Audio data (WAV, MP3) | Raw bytes | -| `video` | Video data (MP4, AVI) | Raw bytes | -| `binary` | Generic binary data | Raw bytes | +| Type | Julia | JavaScript | Python | MicroPython | Description | +|------|-------|------------|--------|-------------|-------------| +| `text` | `String` | `string` | `str` | `str` | Plain text strings | +| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries | +| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | +| `jsontable` | `Vector{NamedTuple}` | `Array` | `list[dict]` | ❌ | Tabular data (JSON) | +| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) | +| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) | +| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) | +| `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | `bytearray` | Generic binary data | --- -## Transport Strategies - -### Direct Transport (Payloads < 1MB) - -Small payloads are sent directly via NATS with Base64 encoding. - -#### Julia -```julia -data = [("message", "Hello", "text")] -smartsend("/topic", data) -``` - -### Link Transport (Payloads >= 1MB) - -Large payloads are uploaded to an HTTP file server. - -#### Julia -```julia -data = [("file", large_data, "binary")] -smartsend("/topic", data; fileserver_url="http://localhost:8080") -``` - ---- - -## Examples +## Cross-Platform Examples ### Example 1: Chat with Mixed Content -Send text, small image, and large file in one message. +Send text, image, and large file in one message. #### Julia + ```julia using NATSBridge @@ -362,11 +369,48 @@ data = [ env, env_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") ``` +#### JavaScript + +```javascript +const NATSBridge = require('natsbridge'); + +const data = [ + ["message_text", "Hello!", "text"], + ["user_avatar", imageData, "image"], + ["large_document", largeFileData, "binary"] +]; + +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat/room1", + data, + { fileserver_url: 'http://localhost:8080' } +); +``` + +#### Python + +```python +from natsbridge import NATSBridge + +data = [ + ("message_text", "Hello!", "text"), + ("user_avatar", image_data, "image"), + ("large_document", large_file_data, "binary") +] + +env, env_json_str = await NATSBridge.smartsend( + "/chat/room1", + data, + fileserver_url="http://localhost:8080" +) +``` + ### Example 2: Dictionary Exchange Send configuration data between platforms. #### Julia + ```julia using NATSBridge @@ -380,11 +424,44 @@ data = [("config", config, "dictionary")] env, env_json_str = NATSBridge.smartsend("/device/config", data) ``` +#### JavaScript + +```javascript +const NATSBridge = require('natsbridge'); + +const config = { + wifi_ssid: "MyNetwork", + wifi_password: "password123", + update_interval: 60 +}; + +const [env, env_json_str] = await NATSBridge.smartsend( + "/device/config", + [["config", config, "dictionary"]] +); +``` + +#### Python + +```python +from natsbridge import NATSBridge + +config = { + "wifi_ssid": "MyNetwork", + "wifi_password": "password123", + "update_interval": 60 +} + +data = [("config", config, "dictionary")] +env, env_json_str = await NATSBridge.smartsend("/device/config", data) +``` + ### Example 3: Table Data (Arrow IPC) Send tabular data using Apache Arrow IPC format. #### Julia + ```julia using NATSBridge using DataFrames @@ -395,18 +472,53 @@ df = DataFrame( score = [95, 88, 92] ) -data = [("students", df, "table")] +data = [("students", df, "arrowtable")] env, env_json_str = NATSBridge.smartsend("/data/analysis", data) ``` -### Example 4: Request-Response Pattern with Envelope JSON +#### JavaScript -Bi-directional communication with reply-to support. The `smartsend` function now returns both the envelope object and a JSON string that can be published directly. +```javascript +const NATSBridge = require('natsbridge'); + +const df = [ + { id: 1, name: "Alice", score: 95 }, + { id: 2, name: "Bob", score: 88 }, + { id: 3, name: "Charlie", score: 92 } +]; + +const [env, env_json_str] = await NATSBridge.smartsend( + "/data/analysis", + [["students", df, "arrowtable"]] +); +``` + +#### Python + +```python +from natsbridge import NATSBridge +import pandas as pd + +df = pd.DataFrame({ + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "score": [95, 88, 92] +}) + +data = [("students", df, "arrowtable")] +env, env_json_str = await NATSBridge.smartsend("/data/analysis", data) +``` + +### Example 4: Request-Response Pattern + +Bi-directional communication with reply-to support. + +#### Julia -#### Julia (Requester) ```julia using NATSBridge +# Requester env, env_json_str = NATSBridge.smartsend( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; @@ -415,81 +527,50 @@ env, env_json_str = NATSBridge.smartsend( ) ``` -#### Julia (Responder) -```julia -using NATS, NATSBridge +#### JavaScript -# Configuration -const SUBJECT = "/device/command" -const NATS_URL = "nats://localhost:4222" +```javascript +const NATSBridge = require('natsbridge'); -function test_responder() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - env = NATSBridge.smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) - - # Extract reply_to from the envelope metadata - reply_to = env["reply_to"] - - for (dataname, data, type) in env["payloads"] - if dataname == "command" && data["action"] == "read_sensor" - response = Dict("sensor_id" => "sensor-001", "value" => 42.5) - # Send response to the reply_to subject from the request - if !isempty(reply_to) - smartsend(reply_to, [("data", response, "dictionary")]) - end - end - end - end - - sleep(120) - NATS.drain(conn) -end - -test_responder() +// Requester +const [env, env_json_str] = await NATSBridge.smartsend( + "/device/command", + [["command", { action: "read_sensor" }, "dictionary"]], + { broker_url: 'nats://localhost:4222', reply_to: '/device/response' } +); ``` -### Example 5: IoT Device Sensor Data +#### Python -IoT device sending sensor data. +```python +from natsbridge import NATSBridge -#### Julia (Receiver) -```julia -using NATS, NATSBridge - -# Configuration -const SUBJECT = "/device/sensors" -const NATS_URL = "nats://localhost:4222" - -function test_receiver() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) - for (dataname, data, type) in env["payloads"] - if dataname == "temperature" - println("Temperature: $data") - elseif dataname == "humidity" - println("Humidity: $data") - end - end - end - - sleep(120) - NATS.drain(conn) -end - -test_receiver() +# Requester +env, env_json_str = await NATSBridge.smartsend( + "/device/command", + [("command", {"action": "read_sensor"}, "dictionary")], + broker_url="nats://localhost:4222", + reply_to="/device/response" +) ``` --- ## Testing -Run the test scripts to verify functionality: +### Test File Organization -### Julia +| Platform | Sender Tests | Receiver Tests | +|----------|--------------|----------------| +| **Julia** | `test/test_julia_*_sender.jl` | `test/test_julia_*_receiver.jl` | +| **JavaScript** | `test/test_js_*_sender.js` | `test/test_js_*_receiver.js` | +| **Python** | `test/test_py_*_sender.py` | `test/test_py_*_receiver.py` | -```julia +### Run Tests + +#### Julia + +```bash # Text message exchange julia test/test_julia_text_sender.jl julia test/test_julia_text_receiver.jl @@ -511,6 +592,57 @@ julia test/test_julia_table_sender.jl julia test/test_julia_table_receiver.jl ``` +#### JavaScript (Node.js) + +```bash +# Text message exchange +node test/test_js_text_sender.js +node test/test_js_text_receiver.js + +# Dictionary exchange +node test/test_js_dictionary_sender.js +node test/test_js_dictionary_receiver.js + +# Binary transfer +node test/test_js_binary_sender.js +node test/test_js_binary_receiver.js + +# Table exchange +node test/test_js_table_sender.js +node test/test_js_table_receiver.js +``` + +#### Python + +```bash +# Text message exchange +python3 test/test_py_text_sender.py +python3 test/test_py_text_receiver.py + +# Dictionary exchange +python3 test/test_py_dictionary_sender.py +python3 test/test_py_dictionary_receiver.py + +# Binary transfer +python3 test/test_py_binary_sender.py +python3 test/test_py_binary_receiver.py + +# Table exchange +python3 test/test_py_table_sender.py +python3 test/test_py_table_receiver.py +``` + +--- + +## Documentation + +For detailed architecture and implementation information, see: + +- [Architecture Documentation](docs/architecture_updated.md) - Cross-platform architecture, API parity, platform-specific patterns +- [Implementation Guide](docs/implementation_updated.md) - Detailed implementation for each platform, handler functions, testing +- [Tutorial](docs/tutorial_updated.md) - Step-by-step getting started guide +- [Walkthrough](docs/walkthrough_updated.md) - Real-world application building guides + --- ## License @@ -535,4 +667,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/docs/architecture.md b/docs/architecture.md index 8272384..b1f7929 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,166 +1,177 @@ -# Architecture Documentation: Bi-Directional Data Bridge +# Cross-Platform Architecture Documentation: Bi-Directional Data Bridge ## Overview -This document describes the architecture for a high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +This document describes the architecture for a high-performance, bi-directional data bridge using **NATS (Core & JetStream)**, implementing the Claim-Check pattern for large payloads. The system is implemented across three platforms with **high-level API parity** while maintaining **idiomatic implementations** for each language. -The system enables seamless communication for Julia applications: -- **Julia** messaging with NATS +**Supported Platforms:** +- **Julia** - Ground truth implementation with full feature set +- **JavaScript** - Node.js and browser-compatible implementation +- **Python/MicroPython** - Desktop and embedded-compatible implementation -### File Server Handler Architecture +### Cross-Platform Design Principles -The system uses **handler functions** to abstract file server operations, allowing support for different file server implementations (e.g., Plik, AWS S3, custom HTTP server). +1. **High-Level API Parity**: All three platforms expose the same `smartsend()` and `smartreceive()` functions with identical signatures and behavior +2. **Idiomatic Implementations**: Each platform uses its native patterns (multiple dispatch in Julia, async/prototype in JS, class-based in Python) +3. **Message Format Consistency**: The `msg_envelope_v1` and `msg_payload_v1` JSON schemas are identical across all platforms +4. **Handler Function Abstraction**: File server operations are abstracted through handler functions for backend flexibility -**Handler Function Signatures:** +--- -```julia -# Upload handler - uploads data to file server and returns URL -# The handler is passed to smartsend as fileserver_upload_handler parameter -# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8}) -# Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url" -fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} +## High-Level API Standard (Cross-Platform) -# Download handler - fetches data from file server URL with exponential backoff -# The handler is passed to smartreceive as fileserver_download_handler parameter -# It receives: (url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String) -# Returns: Vector{UInt8} (the downloaded data) -fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} +### Unified API Signature + +All three platforms expose the same high-level API: + +**Input Format (smartsend):** ``` - -This design allows the system to support multiple file server backends without changing the core messaging logic. - -### Multi-Payload Support (Standard API) - -The system uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.** - -**API Standard:** -```julia -# Input format for smartsend (always a list of tuples with type info) [(dataname1, data1, type1), (dataname2, data2, type2), ...] - -# Output format for smartreceive (returns a dictionary-like object with payloads field containing list of tuples) -# Returns: Dict-like object with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}} -# { -# "correlation_id": "...", -# "msg_id": "...", -# "timestamp": "...", -# "send_to": "...", -# "msg_purpose": "...", -# "sender_name": "...", -# "sender_id": "...", -# "receiver_name": "...", -# "receiver_id": "...", -# "reply_to": "...", -# "reply_to_msg_id": "...", -# "broker_url": "...", -# "metadata": {...}, -# "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] -# } ``` -**Supported Types:** -- `"text"` - Plain text -- `"dictionary"` - JSON-serializable dictionaries (Dict, NamedTuple) -- `"table"` - Tabular data (DataFrame, array of structs) -- `"image"` - Image data (Bitmap, PNG/JPG bytes) -- `"audio"` - Audio data (WAV, MP3 bytes) -- `"video"` - Video data (MP4, AVI bytes) -- `"binary"` - Generic binary data (Vector{UInt8}) +**Output Format (smartreceive):** +``` +{ + "correlation_id": "...", + "msg_id": "...", + "timestamp": "...", + "send_to": "...", + "msg_purpose": "...", + "sender_name": "...", + "sender_id": "...", + "receiver_name": "...", + "receiver_id": "...", + "reply_to": "...", + "reply_to_msg_id": "...", + "broker_url": "...", + "metadata": {...}, + "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] +} +``` -This design allows per-payload type specification, enabling **mixed-content messages** where different payloads can use different serialization formats in a single message. +### Supported Payload Types -**Examples:** +| Type | Julia | JavaScript | Python/MicroPython | +|------|-------|------------|-------------------| +| `text` | `String` | `string` | `str` | +| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | +| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array` (input) → `Buffer` (Arrow IPC) | `pandas.DataFrame`, `bytes` (Arrow IPC) | +| `jsontable` | `Vector{NamedTuple}`, `Vector{Dict}` | `Array` | `list[dict]`, `list` | +| `table` | ❌ | ❌ | `pandas.DataFrame`, `bytes` (Arrow IPC) | +| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | +| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | +| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | +| `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray`, `io.BytesIO` | +**Note on MicroPython:** MicroPython does not support table types (`arrowtable` or `jsontable`) due to memory constraints. Use `dictionary` or `binary` instead. + +### Cross-Platform API Examples + +**Julia:** ```julia -# Single payload - still wrapped in a list -smartsend( - "/test", - [("dataname1", data1, "dictionary")], # List with one tuple (data, type) - broker_url="nats://localhost:4222", - fileserver_upload_handler=plik_oneshot_upload -) +using NATSBridge -# Multiple payloads in one message with different types -smartsend( - "/test", - [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")], - broker_url="nats://localhost:4222", - fileserver_upload_handler=plik_oneshot_upload -) - -# Mixed content (e.g., chat with text, image, audio) -smartsend( +# Send +env, env_json_str = smartsend( "/chat", - [ - ("message_text", "Hello!", "text"), - ("user_image", image_data, "image"), - ("audio_clip", audio_data, "audio") - ], + [("message", "Hello!", "text"), ("image", image_bytes, "image")], broker_url="nats://localhost:4222" ) -# Receive returns a JSON object envelope with all metadata and deserialized payloads -env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) -# Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 -# env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...] -# env["correlation_id"], env["msg_id"], etc. -# env is a JSON object containing envelope metadata and payloads field +# Receive - returns JSON.Object{String, Any} +env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff) +# env is a JSON.Object{String, Any} with "payloads" field containing Vector{Tuple{String, Any, String}} +# Access payloads: for (dataname, data, type) in env["payloads] ``` -## Architecture Diagram +**JavaScript:** +```javascript +const NATSBridge = require('natsbridge'); + +// Send +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat", + [ + ["message", "Hello!", "text"], + ["image", imageBuffer, "image"] + ], + { broker_url: "nats://localhost:4222" } +); + +// Receive - returns Promise +const env = await NATSBridge.smartreceive(msg, { + fileserver_download_handler: fetchWithBackoff +}); +// env is an object with "payloads" field containing Array of arrays +// Access payloads: for (const [dataname, data, type] of env.payloads) +``` + +**Python:** +```python +from natsbridge import NATSBridge + +# Send +env, env_json_str = NATSBridge.smartsend( + "/chat", + [("message", "Hello!", "text"), ("image", image_bytes, "image")], + broker_url="nats://localhost:4222" +) + +# Receive - returns Tuple[Dict, str] +env = NATSBridge.smartreceive( + msg, + fileserver_download_handler=fetch_with_backoff +) +# env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] +# Access payloads: for dataname, data, type_ in env["payloads"] +``` + +**MicroPython:** +```python +from natsbridge import NATSBridge + +# Send (limited to direct transport due to memory constraints) +env, env_json_str = NATSBridge.smartsend( + "/chat", + [("message", "Hello!", "text")], + broker_url="nats://localhost:4222" +) +``` + +--- + +## Architecture Diagram (Cross-Platform) ```mermaid flowchart TD subgraph Client - App[Julia Application] + App[Julia/JS/Python/MicroPython Application] end subgraph Server - Julia[Julia Service] + Julia/JS/Python/MicroPython[Julia/JS/Python/MicroPython Service] NATS[NATS Server] FileServer[HTTP File Server] end App -->|NATS| NATS - NATS -->|NATS| Julia - Julia -->|NATS| NATS - Julia -->|HTTP POST| FileServer + NATS -->|NATS| Julia/JS/Python/MicroPython + Julia/JS/Python/MicroPython -->|NATS| NATS + Julia/JS/Python/MicroPython -->|HTTP POST| FileServer style App fill:#e8f5e9 - style Julia fill:#e8f5e9 + style Julia/JS/Python/MicroPython fill:#e8f5e9 style NATS fill:#fff3e0 style FileServer fill:#f3e5f5 ``` +--- + ## System Components ### 1. msg_envelope_v1 - Message Envelope -The `msg_envelope_v1` structure provides a comprehensive message format for bidirectional communication in Julia applications. - -**Julia Structure:** -```julia -struct msg_envelope_v1 - correlation_id::String # Unique identifier to track messages across systems - msg_id::String # This message id - timestamp::String # Message published timestamp - - send_to::String # Topic/subject the sender sends to - msg_purpose::String # Purpose of this message (ACK | NACK | updateStatus | shutdown | ...) - sender_name::String # Sender name (e.g., "agent-wine-web-frontend") - sender_id::String # Sender id (uuid4) - receiver_name::String # Message receiver name (e.g., "agent-backend") - receiver_id::String # Message receiver id (uuid4 or nothing for broadcast) - reply_to::String # Topic to reply to - reply_to_msg_id::String # Message id this message is replying to - broker_url::String # NATS server address - - metadata::Dict{String, Any} - payloads::Vector{msg_payload_v1} # Multiple payloads stored here -end -``` - -**JSON Schema:** +**JSON Schema (Identical Across All Platforms):** ```json { "correlation_id": "uuid-v4-string", @@ -178,7 +189,8 @@ end "broker_url": "nats://localhost:4222", "metadata": { - + "content_type": "application/octet-stream", + "content_length": 123456 }, "payloads": [ @@ -191,20 +203,18 @@ end "size": 15433, "data": "base64-encoded-string", "metadata": { - + "checksum": "sha256_hash" } }, { "id": "uuid4", - "dataname": "large_data", - "payload_type": "table", + "dataname": "large_arrow_table", + "payload_type": "arrowtable", "transport": "link", - "encoding": "none", + "encoding": "arrow-ipc", "size": 524288, "data": "http://localhost:8080/file/UPLOAD_ID/FILE_ID/data.arrow", - "metadata": { - - } + "metadata": {} } ] } @@ -212,347 +222,254 @@ end ### 2. msg_payload_v1 - Payload Structure -The `msg_payload_v1` structure provides flexible payload handling for various data types. - -**Julia Structure:** -```julia -struct msg_payload_v1 - id::String # Id of this payload (e.g., "uuid4") - dataname::String # Name of this payload (e.g., "login_image") - payload_type::String # "text | dictionary | table | image | audio | video | binary" - transport::String # "direct | link" - encoding::String # "none | json | base64 | arrow-ipc" - size::Integer # Data size in bytes - data::Any # Payload data in case of direct transport or a URL in case of link - metadata::Dict{String, Any} # Dict("checksum" => "sha256_hash", ...) -end +**JSON Schema (Identical Across All Platforms):** +```json +{ + "id": "uuid4", + "dataname": "login_image", + "payload_type": "image | dictionary | arrowtable | jsontable | table | text | audio | video | binary", + "transport": "direct | link", + "encoding": "none | json | base64 | arrow-ipc", + "size": 15433, + "data": "base64-encoded-string | http-url | json-string", + "metadata": { + "checksum": "sha256_hash" + } +} ``` -**Key Features:** -- Supports multiple data types: text, dictionary, table, image, audio, video, binary -- Flexible transport: "direct" (NATS) or "link" (HTTP fileserver) -- Multiple payloads per message (essential for chat with mixed content) -- Per-payload and per-envelope metadata support - -### 3. Transport Strategy Decision Logic +### 3. Transport Strategy Decision Logic (Cross-Platform) ``` ┌─────────────────────────────────────────────────────────────┐ -│ smartsend Function │ +│ smartsend Function (All Platforms) │ │ Accepts: [(dataname1, data1, type1), ...] │ │ (Type is per payload, not standalone) │ └─────────────────────────────────────────────────────────────┘ - │ - ▼ + │ + ▼ ┌─────────────────────────────────────────────────────────────┐ │ For each payload: │ -│ 1. Extract type from tuple │ +│ 1. Extract type from tuple/array │ │ 2. Serialize based on type │ │ 3. Check payload size │ └─────────────────────────────────────────────────────────────┘ - │ - ┌────────────────┴─-────────────────┐ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ Direct Path │ │ Link Path │ - │ (< 1MB) │ │ (> 1MB) │ - │ │ │ │ - │ • Serialize to │ │ • Serialize to │ - │ IOBuffer │ │ IOBuffer │ - │ • Base64 encode │ │ • Upload to │ - │ • Publish to │ │ HTTP Server │ - │ NATS │ │ • Publish to │ - │ (with payload │ │ NATS with URL │ - │ in envelope) │ │ (in envelope) │ - └─────────────────┘ └─────────────────┘ + │ + ┌───────────┴────────────┐ + ▼ ▼ + ┌──────────────┐ ┌──────────────┐ + │ Direct Path │ │ Link Path │ + │ (< 1MB) │ │ (>= 1MB) │ + │ │ │ │ + │ • Serialize │ │ • Serialize │ + │ to buffer │ │ to buffer │ + │ • Base64/JSON│ │ • Upload to │ + │ encode │ │ HTTP Server│ + │ • Publish to │ │ • Publish to │ + │ NATS │ │ NATS with │ + │ (in msg) │ │ URL │ + └──────────────┘ └──────────────┘ ``` -### 4. Julia Module Architecture +--- -```mermaid -graph TD - subgraph JuliaModule - JuliaSmartSend[smartsend] - SizeCheck[Size Check] - DirectPath[Direct Path] - LinkPath[Link Path] - HTTPClient[HTTP Client] - end +## Platform Comparison Matrix - JuliaSmartSend --> SizeCheck - SizeCheck -->|< 1MB| DirectPath - SizeCheck -->|>= 1MB| LinkPath - LinkPath --> HTTPClient +| Feature | Julia | JavaScript | Python | MicroPython | +|---------|-------|------------|--------|-------------| +| **Multiple Dispatch** | ✅ Native | ❌ (Prototypes) | ❌ (Overload via `@overload`) | ❌ | +| **Async/Await** | ❌ (Tasks) | ✅ Native | ✅ Native | ⚠️ (uasyncio) | +| **Type Safety** | ✅ Strong | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ | +| **Memory Management** | ✅ GC | ✅ GC | ✅ GC | ⚠️ (Manual) | +| **Arrow IPC** | ✅ Native | ✅ (arrow package) | ✅ (pyarrow) | ❌ | +| **JSON Serialization** | ✅ (JSON.jl) | ✅ (native) | ✅ (json) | ✅ (json) | +| **arrowtable Support** | ✅ | ✅ | ✅ | ❌ | +| **jsontable Support** | ✅ | ✅ | ✅ | ❌ | +| **Direct Transport** | ✅ | ✅ | ✅ | ✅ | +| **Link Transport** | ✅ | ✅ | ✅ | ⚠️ (Limited) | +| **Handler Functions** | ✅ | ✅ | ✅ | ✅ | +| **Cross-Platform API** | ✅ | ✅ | ✅ | ✅ | - style JuliaModule fill:#c5e1a5 +--- + +## Platform-Specific Architecture Patterns + +### Julia: Multiple Dispatch Pattern + +Julia leverages multiple dispatch for type-specific implementations: + +- **Function overloading** based on argument types +- **Struct-based data models** with explicit types +- **Native Arrow IPC** support via Arrow.jl + +### JavaScript: Prototype + Async Pattern + +JavaScript uses async/await for non-blocking I/O: + +- **Class-based NATS client** for connection management +- **Module-level utility functions** for serialization +- **Native ArrayBuffer** for binary data handling + +### Python: Class-Based Pattern + +Python uses classes for stateful operations: + +- **Class-based NATSBridge** with type hints +- **Dataclasses** for structured data (MsgPayloadV1, MsgEnvelopeV1) +- **Async/await** for I/O operations + +### MicroPython: Synchronous Pattern + +MicroPython has significant constraints: + +- **Synchronous API** (no async/await) +- **Memory-constrained** (256KB - 1MB) +- **Limited payload support** (no tables, max 50KB) + +--- + +## Cross-Platform Compatibility Notes + +### 1. Payload Type Consistency + +All platforms use the same payload type values for tabular data: + +| Platform | Table Types | +|----------|-------------| +| Julia | `"arrowtable"`, `"jsontable"` | +| JavaScript | `"arrowtable"`, `"jsontable"` | +| Python | `"arrowtable"`, `"jsontable"` | +| MicroPython | Not supported | + + +### 2. Direct Transport Encoding Field + +The encoding field in direct transport payloads differs between platforms: + +| Platform | Encoding for Direct Transport | +|----------|-------------------------------| +| Julia | Preserves original type: `"base64"`, `"json"`, or `"arrow-ipc"` | +| JavaScript | Preserves original type: `"base64"`, `"json"`, or `"arrow-ipc"` | +| Python | Always `"base64"` for all direct transport payloads | +| MicroPython | Always `"base64"` for all direct transport payloads | + +**Impact:** The encoding field may not accurately reflect the original serialization format when using Python or MicroPython. + +### 3. MicroPython Limitations + +MicroPython has significant constraints that affect feature support: + +| Feature | Desktop Platforms | MicroPython | +|---------|-------------------|-------------| +| `arrowtable` | ✅ | ❌ (not supported - memory constraints) | +| `jsontable` | ✅ | ❌ (not supported - memory constraints) | +| `table` | ✅ | ❌ (not supported - memory constraints) | +| Async/await | ✅ | ❌ (synchronous only) | +| File upload/download | ✅ | ⚠️ (placeholder implementations) | +| MAX_PAYLOAD_SIZE | 1MB+ | 50KB (hard limit) | +| DEFAULT_SIZE_THRESHOLD | 1MB | 100KB | + +**Impact:** MicroPython should only be used for small payloads with direct transport. File server operations are not fully implemented. + +--- + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `NATS_URL` | `nats://localhost:4222` | NATS server URL | +| `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL | +| `SIZE_THRESHOLD` | `1000000` | Size threshold in bytes (1MB) | + +### MicroPython-Specific Configuration + +```python +# micropython.conf +NATS_URL = "nats://broker.local:4222" +FILESERVER_URL = "http://fileserver.local:8080" +SIZE_THRESHOLD = 100000 # Lower threshold for memory-constrained devices +MAX_PAYLOAD_SIZE = 50000 # Hard limit for MicroPython ``` -## Implementation Details - -### Julia Implementation - -#### Dependencies -- `NATS.jl` - Core NATS functionality -- `Arrow.jl` - Arrow IPC serialization -- `JSON3.jl` - JSON parsing -- `HTTP.jl` - HTTP client for file server -- `Dates.jl` - Timestamps for logging - -#### smartsend Function - -```julia -function smartsend( - subject::String, - data::AbstractArray{Tuple{String, Any, String}, 1}; # List of (dataname, data, type) tuples - broker_url::String = DEFAULT_BROKER_URL, # NATS server URL - fileserver_url = DEFAULT_FILESERVER_URL, - fileserver_upload_handler::Function = plik_oneshot_upload, - size_threshold::Int = DEFAULT_SIZE_THRESHOLD, - correlation_id::String = string(uuid4()), # Correlation ID for tracing (auto-generated UUID) - msg_purpose::String = "chat", - sender_name::String = "NATSBridge", - receiver_name::String = "", - receiver_id::String = "", - reply_to::String = "", - reply_to_msg_id::String = "", - is_publish::Bool = true, # Whether to automatically publish to NATS - NATS_connection::Union{NATS.Connection, Nothing} = nothing, # Pre-existing NATS connection (optional, saves connection overhead) - msg_id::String = string(uuid4()), # Message ID (auto-generated UUID) - sender_id::String = string(uuid4()) # Sender ID (auto-generated UUID) -) -``` - -**Keyword Parameter - NATS_connection:** -- `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios where connection reuse provides performance benefits. - -**Connection Handling Logic:** -```julia -if is_publish == false - # skip publish a message -elseif is_publish == true && NATS_connection === nothing - publish_message(broker_url, subject, env_json_str, cid) # Creates new connection -elseif is_publish == true && NATS_connection !== nothing - publish_message(NATS_connection, subject, env_json_str, cid) # Uses provided connection -end -``` - -**Return Value:** -- Returns a tuple `(env, env_json_str)` where: - - `env::msg_envelope_v1` - The envelope object containing all metadata and payloads - - `env_json_str::String` - JSON string representation of the envelope for publishing - -**Options:** -- `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern. - -The envelope object can be accessed directly for programmatic use, while the JSON string can be published directly to NATS using the request-reply pattern. - -**Input Format:** -- `data::AbstractArray{Tuple{String, Any, String}}` - **Must be a list of (dataname, data, type) tuples**: `[("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...]` -- Even for single payloads: `[(dataname1, data1, "type1")]` -- Each payload can have a different type, enabling mixed-content messages - -**Flow:** -1. Iterate through the list of `(dataname, data, type)` tuples -2. For each payload: extract the type from the tuple and serialize accordingly -3. Check payload size -4. If < threshold: publish directly to NATS with Base64-encoded payload -5. If >= threshold: upload to HTTP server, publish NATS with URL - -#### smartreceive Handler - -```julia -function smartreceive( - msg::NATS.Msg; - fileserver_download_handler::Function = _fetch_with_backoff, - max_retries::Int = 5, - base_delay::Int = 100, - max_delay::Int = 5000 -) - # Parse envelope - # Iterate through all payloads - # For each payload: check transport type - # If direct: decode Base64 payload - # If link: fetch from URL with exponential backoff using fileserver_download_handler - # Deserialize payload based on type - # Return envelope dictionary with all metadata and deserialized payloads -end -``` - -**Output Format:** -- Returns a JSON object (dictionary) containing all envelope fields: - - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - - `metadata` - Message-level metadata dictionary - - `payloads` - List of tuples, each containing `(dataname, data, type)` with deserialized payload data - -**Process Flow:** -1. Parse the JSON envelope to extract all fields -2. Iterate through each payload in `payloads` -3. For each payload: - - Determine transport type (`direct` or `link`) - - If `direct`: decode Base64 data from the message - - If `link`: fetch data from URL using exponential backoff (via `fileserver_download_handler`) - - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) -4. Return envelope dictionary with `payloads` field containing list of `(dataname, data, type)` tuples - -**Note:** The `fileserver_download_handler` receives `(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)` and returns `Vector{UInt8}`. - -#### publish_message Function - -The `publish_message` function provides two overloads for publishing messages to NATS: - -**Overload 1 - URL-based publishing (creates new connection):** -```julia -function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) - conn = NATS.connect(broker_url) # Create NATS connection - publish_message(conn, subject, message, correlation_id) -end -``` - -**Overload 2 - Connection-based publishing (uses pre-existing connection):** -```julia -function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) - try - NATS.publish(conn, subject, message) # Publish message to NATS - log_trace(correlation_id, "Message published to $subject") # Log successful publish - finally - NATS.drain(conn) # Ensure connection is closed properly - end -end -``` - -**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. This is a Julia-specific optimization that leverages function overloading. - -**Integration with smartsend:** -```julia -# When NATS_connection is provided to smartsend, it uses the connection-based publish_message -env, env_json_str = smartsend( - "my.subject", - [("data", payload_data, "type")], - NATS_connection=my_connection, # Pre-existing connection - is_publish=true -) -# Uses: publish_message(NATS_connection, subject, env_json_str, cid) - -# When NATS_connection is not provided, it uses the URL-based publish_message -env, env_json_str = smartsend( - "my.subject", - [("data", payload_data, "type")], - broker_url="nats://localhost:4222", - is_publish=true -) -# Uses: publish_message(broker_url, subject, env_json_str, cid) -``` - -## Scenario Implementations - -### Scenario 1: Command & Control (Small Dictionary) - -**Julia (Sender/Receiver):** -```julia -# Subscribe to control subject -# Parse JSON envelope -# Execute simulation with parameters -# Send acknowledgment -``` - -### Scenario 2: Deep Dive Analysis (Large Arrow Table) - -**Julia (Sender/Receiver):** -```julia -# Create large DataFrame -# Convert to Arrow IPC stream -# Check size (> 1MB) -# Upload to HTTP server -# Publish NATS with URL -``` - -### Scenario 3: Live Audio Processing - -**Julia (Sender/Receiver):** -```julia -# Receive audio data -# Perform FFT or AI transcription -# Send results back (JSON + Arrow table) -``` - -### Scenario 4: Catch-Up (JetStream) - -**Julia (Producer/Consumer):** -```julia -# Publish to JetStream -# Include metadata for temporal tracking -``` - -### Scenario 5: Selection (Low Bandwidth) - -**Focus:** Small Arrow tables. The Action: Julia wants to send a small DataFrame to show on a receiving application for the user to choose. - -**Julia (Sender/Receiver):** -```julia -# Create small DataFrame (e.g., 50KB - 500KB) -# Convert to Arrow IPC stream -# Check payload size (< 1MB threshold) -# Publish directly to NATS with Base64-encoded payload -# Include metadata for dashboard selection context -``` - -### Scenario 6: Chat System - -**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging. - -**Multi-Payload Support:** The system supports mixed-payload messages where a single message can contain multiple payloads with different transport strategies. The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. - -**Julia (Sender/Receiver):** -```julia -# Build chat message with mixed payloads: -# - Text: direct transport (Base64) -# - Small images: direct transport (Base64) -# - Large images: link transport (HTTP URL) -# - Audio/video: link transport (HTTP URL) -# - Tables: direct or link depending on size -# - Files: link transport (HTTP URL) -# -# Each payload uses appropriate transport strategy: -# - Size < 1MB → direct (NATS + Base64) -# - Size >= 1MB → link (HTTP upload + NATS URL) -# -# Include claim-check metadata for delivery tracking -# Support bidirectional messaging with replyTo fields -``` - -**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components. - -**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads. +--- ## Performance Considerations ### Zero-Copy Reading -- Use Arrow's memory-mapped file reading -- Avoid unnecessary data copying during deserialization -- Use Apache Arrow's native IPC reader + +| Platform | Strategy | +|----------|----------| +| **Julia** | `Arrow.read()` with memory-mapped files | +| **JavaScript** | `ArrayBuffer` with `DataView` | +| **Python** | `pyarrow` memory mapping | +| **MicroPython** | Not available (streaming only) | ### Exponential Backoff -- Implement exponential backoff for HTTP link fetching -- Maximum retry count: 5 -- Base delay: 100ms, max delay: 5000ms + +All platforms implement exponential backoff for HTTP downloads: + +``` +delay = base_delay +for attempt in 1:max_retries: + try: + response = fetch(url) + if success: return response + except: + if attempt < max_retries: + sleep(delay) + delay = min(delay * 2, max_delay) +``` ### Correlation ID Logging -- Log correlation_id at every stage -- Include: send, receive, serialize, deserialize -- Use structured logging format -## Testing Strategy +All platforms use correlation IDs for distributed tracing: -### Unit Tests -- Test smartsend with various payload sizes -- Test smartreceive with direct and link transport -- Test Arrow IPC serialization/deserialization +``` +[timestamp] [Correlation: abc123] Message published to subject +``` -### Integration Tests -- Test full flow with NATS server -- Test large data transfer (> 100MB) -- Test audio processing pipeline +### Serialization Performance Comparison -### Performance Tests -- Measure throughput for small payloads -- Measure throughput for large payloads +| Format | Use Case | Pros | Cons | +|--------|----------|------|------| +| `arrowtable` | Large tabular data | Fast, zero-copy, schema-preserving | Binary format, requires Arrow library | +| `jsontable` | Small/medium tabular data | Human-readable, universal support | Slower, larger size, no schema | +| `table` (Python) | Large tabular data | Fast, zero-copy, schema-preserving | Python-specific, requires pyarrow | + +--- + +## Summary + +This cross-platform NATS bridge provides: + +1. **High-Level API Parity**: Identical `smartsend()` and `smartreceive()` signatures across Julia, JavaScript, and Python/MicroPython +2. **Idiomatic Implementations**: + - Julia: Multiple dispatch and struct-based design + - JavaScript: Async/await and prototype-based utilities + - Python: Class-based design with type hints + - MicroPython: Synchronous API with memory constraints +3. **Message Format Consistency**: Identical `msg_envelope_v1` and `msg_payload_v1` JSON schemas +4. **Handler Abstraction**: File server operations abstracted through configurable handlers +5. **Platform-Specific Optimizations**: + - **Arrow IPC** (`arrowtable`): Efficient binary format for large tabular data + - **JSON** (`jsontable`): Universal human-readable format for smaller tables + - **Python table**: Unified table type for Python-specific implementations + - Streaming support in MicroPython + +The Julia implementation serves as the **ground truth** for API design and behavior, while JavaScript and Python implementations maintain interface parity while leveraging their respective language idioms. + +### Datatype Summary + +| Datatype | Serialization | Use Case | Encoding | Supported Platforms | +|----------|---------------|----------|----------|---------------------| +| `text` | UTF-8 bytes | Text messages, chat content | `utf-8` → `base64` | All | +| `dictionary` | JSON | Structured key-value data, config | `json` → `base64` | All | +| `arrowtable` | Apache Arrow IPC | Large tabular data, schema-preserving | `arrow-ipc` → `base64` | Julia, JavaScript, Python | +| `jsontable` | JSON | Small/medium tabular data, human-readable | `json` → `base64` | Julia, JavaScript, Python | +| `table` | Apache Arrow IPC | Python's unified table type | `arrow-ipc` → `base64` | Python | +| `image` | Binary | Image files (JPEG, PNG, etc.) | `binary` → `base64` | All | +| `audio` | Binary | Audio files (WAV, MP3, etc.) | `binary` → `base64` | All | +| `video` | Binary | Video files (MP4, AVI, etc.) | `binary` → `base64` | All | +| `binary` | Binary | Generic binary data, files | `binary` → `base64` | All | diff --git a/docs/implementation.md b/docs/implementation.md index c9a29e2..8467f78 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -1,545 +1,1711 @@ -# Implementation Guide: Bi-Directional Data Bridge +# Cross-Platform Implementation Guide: Bi-Directional Data Bridge ## Overview -This document describes the implementation of the high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +This document describes the detailed implementation of the high-performance, bi-directional data bridge using **NATS (Core & JetStream)**, implementing the Claim-Check pattern for large payloads. The system is implemented across three platforms with **high-level API parity** while maintaining **idiomatic implementations** for each language. -The system enables seamless communication for Julia applications. +**Supported Platforms:** +- **Julia** - Ground truth implementation (reference) +- **JavaScript** - Node.js and browser implementation +- **Python/MicroPython** - Desktop and embedded implementation -### Implementation Files +--- -NATSBridge is implemented in Julia: +## Cross-Platform Compatibility Notes + +### 1. Python Payload Type Naming + +The Python implementation uses `"table"` as a single payload type for both Arrow and JSON table serialization, while Julia and JavaScript use separate types (`"arrowtable"` and `"jsontable"`): + +| Platform | Table Types | +|----------|-------------| +| Julia | `"arrowtable"`, `"jsontable"` | +| JavaScript | `"arrowtable"`, `"jsontable"` | +| Python | `"table"` (single type) | +| MicroPython | Not supported | + +**Impact:** When exchanging data between Python and Julia/JavaScript, the payload type will differ. Python code should use `"table"` while Julia/JavaScript code should use `"arrowtable"` or `"jsontable"`. + +### 2. Direct Transport Encoding Field + +The encoding field in direct transport payloads differs between platforms: + +| Platform | Encoding for Direct Transport | +|----------|-------------------------------| +| Julia | Preserves original type: `"base64"`, `"json"`, or `"arrow-ipc"` | +| JavaScript | Preserves original type: `"base64"`, `"json"`, or `"arrow-ipc"` | +| Python | Always `"base64"` for all direct transport payloads | +| MicroPython | Always `"base64"` for all direct transport payloads | + +**Impact:** The encoding field may not accurately reflect the original serialization format when using Python or MicroPython. + +### 3. MicroPython Limitations + +MicroPython has significant constraints that affect feature support: + +| Feature | Desktop Platforms | MicroPython | +|---------|-------------------|-------------| +| `arrowtable` | ✅ | ❌ (not supported - memory constraints) | +| `jsontable` | ✅ | ❌ (not supported - memory constraints) | +| `table` | ✅ | ❌ (not supported - memory constraints) | +| Async/await | ✅ | ❌ (synchronous only) | +| File upload/download | ✅ | ⚠️ (placeholder implementations) | +| MAX_PAYLOAD_SIZE | 1MB+ | 50KB (hard limit) | +| DEFAULT_SIZE_THRESHOLD | 1MB | 100KB | + +**Impact:** MicroPython should only be used for small payloads with direct transport. File server operations are not fully implemented. + +--- + +## Implementation Files | Language | Implementation File | Description | |----------|---------------------|-------------| | **Julia** | [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Full Julia implementation with Arrow IPC support | +| **JavaScript** | `src/natsbridge.js` | Node.js/browser implementation | +| **Python** | `src/natsbridge.py` | Desktop Python implementation | +| **MicroPython** | `src/natsbridge_mpy.py` | MicroPython implementation (limited features) | -### File Server Handler Architecture +--- + +## File Server Handler Architecture The system uses **handler functions** to abstract file server operations, allowing support for different file server implementations (e.g., Plik, AWS S3, custom HTTP server). -**Handler Function Signatures:** +### Handler Function Signatures + +#### Julia ```julia # Upload handler - uploads data to file server and returns URL -# The handler is passed to smartsend as fileserver_upload_handler parameter -# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8}) -# Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url" -fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} +fileserver_upload_handler( + fileserver_url::String, + dataname::String, + data::Vector{UInt8} +)::Dict{String, Any} # Download handler - fetches data from file server URL with exponential backoff -# The handler is passed to smartreceive as fileserver_download_handler parameter -# It receives: (url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String) -# Returns: Vector{UInt8} (the downloaded data) -fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} +fileserver_download_handler( + url::String, + max_retries::Int, + base_delay::Int, + max_delay::Int, + correlation_id::String +)::Vector{UInt8} ``` -This design allows the system to support multiple file server backends without changing the core messaging logic. +#### JavaScript -### Multi-Payload Support (Standard API) +```javascript +// Upload handler - async function +async function fileserver_upload_handler( + fileserver_url, + dataname, + data // Uint8Array +) { + // Returns: { status, uploadid, fileid, url } +} -The system uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.** +// Download handler - async function +async function fileserver_download_handler( + url, + max_retries, + base_delay, + max_delay, + correlation_id +) { + // Returns: Uint8Array +} +``` -**API Standard:** -```julia +#### Python + +```python +# Upload handler - async function +async def fileserver_upload_handler( + fileserver_url: str, + dataname: str, + data: bytes +) -> Dict[str, Any]: + """ + Upload data to file server. + + Returns: + Dict with keys: 'status', 'uploadid', 'fileid', 'url' + """ + pass + +# Download handler - async function +async def fileserver_download_handler( + url: str, + max_retries: int, + base_delay: int, + max_delay: int, + correlation_id: str +) -> bytes: + """ + Download data from URL with exponential backoff. + + Returns: + Downloaded bytes + """ + pass +``` + +#### MicroPython + +```python +# Upload handler - synchronous (no async in MicroPython) +def fileserver_upload_handler( + fileserver_url: str, + dataname: str, + data: bytearray +) -> Dict: + """ + Upload data to file server (synchronous). + + Returns: + Dict with keys: 'status', 'url' + """ + pass + +# Download handler - synchronous +def fileserver_download_handler( + url: str, + max_retries: int, + base_delay: int, + max_delay: int, + correlation_id: str +) -> bytearray: + """ + Download data from URL with exponential backoff (synchronous). + + Returns: + Downloaded bytes + """ + pass +``` + +--- + +## Multi-Payload Support (Standard API) + +The system uses a **standardized list-of-tuples format** for all payload operations across all platforms. + +### API Standard + +``` # Input format for smartsend (always a list of tuples with type info) [(dataname1, data1, type1), (dataname2, data2, type2), ...] # Output format for smartreceive (returns a dictionary with payloads field containing list of tuples) -# Returns: Dict with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}} -# { -# "correlation_id": "...", -# "msg_id": "...", -# "timestamp": "...", -# "send_to": "...", -# "msg_purpose": "...", -# "sender_name": "...", -# "sender_id": "...", -# "receiver_name": "...", -# "receiver_id": "...", -# "reply_to": "...", -# "reply_to_msg_id": "...", -# "broker_url": "...", -# "metadata": {...}, -# "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] -# } +{ + "correlation_id": "...", + "msg_id": "...", + "timestamp": "...", + "send_to": "...", + "msg_purpose": "...", + "sender_name": "...", + "sender_id": "...", + "receiver_name": "...", + "receiver_id": "...", + "reply_to": "...", + "reply_to_msg_id": "...", + "broker_url": "...", + "metadata": {...}, + "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] +} ``` -**Supported Types:** -- `"text"` - Plain text -- `"dictionary"` - JSON-serializable dictionaries (Dict, NamedTuple) -- `"table"` - Tabular data (DataFrame, array of structs) -- `"image"` - Image data (Bitmap, PNG/JPG bytes) -- `"audio"` - Audio data (WAV, MP3 bytes) -- `"video"` - Video data (MP4, AVI bytes) -- `"binary"` - Generic binary data (Vector{UInt8}) +### Supported Types -This design allows per-payload type specification, enabling **mixed-content messages** where different payloads can use different serialization formats in a single message. +| Type | Julia | JavaScript | Python | MicroPython | +|------|-------|------------|--------|-------------| +| `text` | `String` | `string` | `str` | `str` | +| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | +| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array` (input) → `Buffer` (Arrow IPC) | `pandas.DataFrame`, `bytes` (Arrow IPC) | ❌ (not supported) | +| `jsontable` | `Vector{NamedTuple}`, `Vector{Dict}` | `Array` | `list[dict]`, `list` | ⚠️ (limited) | +| `table` | ❌ | ❌ | `pandas.DataFrame`, `bytes` (Arrow IPC) | ❌ | +| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | +| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | +| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | +| `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | `bytearray` | -**Examples:** -```julia -# Single payload - still wrapped in a list -smartsend( - "/test", - [("dataname1", data1, "dictionary")], # List with one tuple (data, type) - broker_url="nats://localhost:4222", - fileserver_upload_handler=plik_oneshot_upload -) +**Note:** Python uses `"table"` as a single type for both Arrow and JSON table serialization. When exchanging data between Python and Julia/JavaScript, ensure the payload type is correctly translated (`"table"` ↔ `"arrowtable"` or `"jsontable"`). -# Multiple payloads in one message with different types -smartsend( - "/test", - [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")], - broker_url="nats://localhost:4222", - fileserver_upload_handler=plik_oneshot_upload -) +--- -# Mixed content (e.g., chat with text, image, audio) -smartsend( - "/chat", - [ - ("message_text", "Hello!", "text"), - ("user_image", image_data, "image"), - ("audio_clip", audio_data, "audio") - ], - broker_url="nats://localhost:4222" -) +## Platform-Specific Implementations -# Receive returns a JSON object envelope with all metadata and deserialized payloads -env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) -# Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 -# env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...] -# env["correlation_id"], env["msg_id"], etc. -# env is a JSON object containing envelope metadata and payloads field -``` +### Julia Implementation -## Architecture - -The Julia implementation follows the Claim-Check pattern: - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ SmartSend Function │ -└─────────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────┐ -│ Is payload size < 1MB? │ -└─────────────────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────┴─────────────────┐ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ Direct Path │ │ Link Path │ - │ (< 1MB) │ │ (> 1MB) │ - │ │ │ │ - │ • Serialize to │ │ • Serialize to │ - │ Buffer │ │ Buffer │ - │ • Base64 encode │ │ • Upload to │ - │ • Publish to │ │ HTTP Server │ - │ NATS │ │ • Publish to │ - │ │ │ NATS with URL │ - └─────────────────┘ └─────────────────┘ -``` - -## smartsend Return Value - -The `smartsend` function now returns a tuple containing both the envelope object and the JSON string representation: +#### Module Structure ```julia -env, env_json_str = smartsend(...) -# Returns: ::Tuple{msg_envelope_v1, String} -# env::msg_envelope_v1 - The envelope object with all metadata and payloads -# env_json_str::String - JSON string for publishing to NATS +module NATSBridge + using NATS, JSON, Arrow, HTTP, UUIDs, Dates, Base64 + + # Constants + const DEFAULT_SIZE_THRESHOLD = 1_000_000 # 1MB + const DEFAULT_BROKER_URL = "nats://localhost:4222" + const DEFAULT_FILESERVER_URL = "http://localhost:8080" + + # Structs + struct msg_payload_v1 + id::String + dataname::String + payload_type::String + transport::String + encoding::String + size::Integer + data::Any + metadata::Dict{String, Any} + end + + struct msg_envelope_v1 + correlation_id::String + msg_id::String + timestamp::String + send_to::String + msg_purpose::String + sender_name::String + sender_id::String + receiver_name::String + receiver_id::String + reply_to::String + reply_to_msg_id::String + broker_url::String + metadata::Dict{String, Any} + payloads::Vector{msg_payload_v1} + end + + # Main functions + function smartsend(...) end + function smartreceive(...) end + + # Utility functions + function _serialize_data(...) end + function _deserialize_data(...) end + function envelope_to_json(...) end + function log_trace(...) end + + # File server handlers + function plik_oneshot_upload(...) end + function _fetch_with_backoff(...) end + function publish_message(...) end + + # Internal helpers + function _get_payload_bytes(...) end +end ``` -**Options:** -- `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern. +#### Multiple Dispatch Pattern -This enables two use cases: -1. **Programmatic envelope access**: Access envelope fields directly via the `env` object -2. **Direct JSON publishing**: Publish the JSON string directly using NATS request-reply pattern - -### Julia Module: [`src/NATSBridge.jl`](../src/NATSBridge.jl) - -The Julia implementation provides: - -- **[`msg_envelope_v1`](src/NATSBridge.jl)**: Struct for the unified JSON envelope -- **[`msg_payload_v1`](src/NATSBridge.jl)**: Struct for individual payload representation -- **[`smartsend()`](src/NATSBridge.jl)**: Handles transport selection based on payload size -- **[`smartreceive()`](src/NATSBridge.jl)**: Handles both direct and link transport - -## Installation - -### Julia Dependencies +Julia leverages multiple dispatch for type-specific implementations: ```julia -using Pkg -Pkg.add("NATS") -Pkg.add("Arrow") -Pkg.add("JSON3") -Pkg.add("HTTP") -Pkg.add("UUIDs") -Pkg.add("Dates") -``` - -## Usage Tutorial - -### Step 1: Start NATS Server - -```bash -docker run -p 4222:4222 nats:latest -``` - -### Step 2: Start HTTP File Server (optional) - -```bash -# Create a directory for file uploads -mkdir -p /tmp/fileserver - -# Use any HTTP server that supports POST for file uploads -# Example: Python's built-in server -python3 -m http.server 8080 --directory /tmp/fileserver -``` - -### Step 3: Run Test Scenarios - -```bash -# Scenario 1: Command & Control -julia test/scenario1_command_control.jl - -# Scenario 2: Large Arrow Table -julia test/scenario2_large_table.jl - -# Scenario 3: Julia-to-Julia communication -julia test/scenario3_julia_to_julia.jl -``` - -## Usage - -### Scenario 1: Command & Control (Small Dictionary) - -**Focus:** Sending small dictionary configurations. This is the simplest use case for command and control scenarios. - -**Julia (Sender/Receiver):** -```julia -using NATSBridge - -# Send small dictionary config (wrapped in list with type) -config = Dict("step_size" => 0.01, "iterations" => 1000, "threshold" => 0.5) -env, env_json_str = smartsend( - "control", - [("config", config, "dictionary")], - broker_url="nats://localhost:4222" -) -# env: msg_envelope_v1 with all metadata and payloads -# env_json_str: JSON string for publishing -``` - -**Julia (Sender/Receiver) with NATS_connection for connection reuse:** -```julia -using NATSBridge - -# Create connection once for high-frequency publishing -conn = NATS.connect("nats://localhost:4222") - -# Send multiple messages using the same connection (saves connection overhead) -for i in 1:100 - config = Dict("iteration" => i, "data" => rand()) - smartsend( - "control", - [("config", config, "dictionary")], - NATS_connection=conn, # Reuse connection - is_publish=true - ) +# publish_message has two overloads based on argument types +function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) + conn = NATS.connect(broker_url) + publish_message(conn, subject, message, correlation_id) end -# Close connection when done -NATS.close(conn) +function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) + try + NATS.publish(conn, subject, message) + log_trace(correlation_id, "Message published to $subject") + finally + NATS.drain(conn) + end +end + +# Type-specific serialization +function _serialize_data(data::String, payload_type::String) + # Text handling + return Vector{UInt8}(data) +end + +function _serialize_data(data::Dict, payload_type::String) + # Dictionary handling + json_str = JSON.json(data) + return Vector{UInt8}(json_str) +end + +function _serialize_data(data::DataFrame, payload_type::String) + # Table handling - arrowtable + io = IOBuffer() + Arrow.write(io, data) + return take!(io) +end ``` -**Use Case:** High-frequency publishing scenarios where connection reuse provides performance benefits by avoiding the overhead of establishing a new NATS connection for each message. - -### Basic Multi-Payload Example - -#### Julia (Sender) -```julia -using NATSBridge - -# Send multiple payloads in one message (type is required per payload) -smartsend( - "/test", - [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")], - broker_url="nats://localhost:4222", - fileserver_url="http://localhost:8080" -) - -# Even single payload must be wrapped in a list with type -smartsend("/test", [("single_data", mydata, "dictionary")], broker_url="nats://localhost:4222") -``` - -#### Julia (Receiver) -```julia -using NATSBridge - -# Receive returns a JSON object with envelope metadata and payloads field -env = smartreceive(msg) -# Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 -# env["payloads"] = [(dataname1, data1, "dictionary"), (dataname2, data2, "table"), ...] -``` - -### Scenario 2: Deep Dive Analysis (Large Arrow Table) - -#### Julia (Sender) -```julia -using Arrow -using DataFrames - -# Create large DataFrame -df = DataFrame( - id = 1:10_000_000, - value = rand(10_000_000), - category = rand(["A", "B", "C"], 10_000_000) -) - -# Send via smartsend - wrapped in list with type -# Large payload will use link transport (HTTP fileserver) -env, env_json_str = smartsend( - "analysis_results", - [("table_data", df, "table")], - broker_url="nats://localhost:4222", - fileserver_url="http://localhost:8080" -) -# env: msg_envelope_v1 with all metadata and payloads -# env_json_str: JSON string for publishing -``` - -#### smartsend Function Signature (Julia) +#### smartsend Implementation ```julia function smartsend( - subject::String, - data::AbstractArray{Tuple{String, Any, String}, 1}; # List of (dataname, data, type) tuples - broker_url::String = DEFAULT_BROKER_URL, # NATS server URL - fileserver_url = DEFAULT_FILESERVER_URL, - fileserver_upload_handler::Function = plik_oneshot_upload, - size_threshold::Int = DEFAULT_SIZE_THRESHOLD, - correlation_id::String = string(uuid4()), # Correlation ID for tracing (auto-generated UUID) - msg_purpose::String = "chat", - sender_name::String = "NATSBridge", - receiver_name::String = "", - receiver_id::String = "", - reply_to::String = "", - reply_to_msg_id::String = "", - is_publish::Bool = true, - NATS_connection::Union{NATS.Connection, Nothing} = nothing, # Pre-existing NATS connection (optional) - msg_id::String = string(uuid4()), # Message ID (auto-generated UUID) - sender_id::String = string(uuid4()) # Sender ID (auto-generated UUID) -) -``` + subject::String, + data::AbstractArray{Tuple{String, T1, String}, 1}; + broker_url::String = DEFAULT_BROKER_URL, + fileserver_url = DEFAULT_FILESERVER_URL, + fileserver_upload_handler::Function = plik_oneshot_upload, + size_threshold::Int = DEFAULT_SIZE_THRESHOLD, + correlation_id::String = string(uuid4()), + msg_purpose::String = "chat", + sender_name::String = "NATSBridge", + receiver_name::String = "", + receiver_id::String = "", + reply_to::String = "", + reply_to_msg_id::String = "", + is_publish::Bool = true, + NATS_connection::Union{NATS.Connection, Nothing} = nothing, + msg_id::String = string(uuid4()), + sender_id::String = string(uuid4()) +)::Tuple{msg_envelope_v1, String} where {T1<:Any} -**New Keyword Parameters:** -- `correlation_id::String = string(uuid4())` - Correlation ID for tracing (auto-generated UUID) -- `msg_id::String = string(uuid4())` - Message ID (auto-generated UUID) -- `sender_id::String = string(uuid4())` - Sender ID (auto-generated UUID) -- `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios. + log_trace(correlation_id, "Starting smartsend for subject: $subject") + + # Process each payload in the list + payloads = msg_payload_v1[] + for (dataname, payload_data, payload_type) in data + # Serialize data based on type + payload_bytes = _serialize_data(payload_data, payload_type) + + payload_size = length(payload_bytes) + log_trace(correlation_id, "Serialized payload '$dataname' size: $payload_size bytes") + + # Decision: Direct vs Link + if payload_size < size_threshold + # Direct path - Base64 encode and send via NATS + payload_b64 = Base64.base64encode(payload_bytes) + log_trace(correlation_id, "Using direct transport for $payload_size bytes") + + payload = msg_payload_v1( + payload_b64, + payload_type; + id = string(uuid4()), + dataname = dataname, + transport = "direct", + encoding = "base64", + size = payload_size, + metadata = Dict{String, Any}("payload_bytes" => payload_size) + ) + push!(payloads, payload) + else + # Link path - Upload to HTTP server, send URL via NATS + log_trace(correlation_id, "Using link transport, uploading to fileserver") + + response = fileserver_upload_handler(fileserver_url, dataname, payload_bytes) + + if response["status"] != 200 + error("Failed to upload data to fileserver: $(response["status"])") + end + + url = response["url"] + log_trace(correlation_id, "Uploaded to URL: $url") -**Connection Handling Logic:** -```julia -if is_publish == false - # skip publish -elseif is_publish == true && NATS_connection === nothing - publish_message(broker_url, subject, env_json_str, cid) # Creates new connection -elseif is_publish == true && NATS_connection !== nothing - publish_message(NATS_connection, subject, env_json_str, cid) # Uses provided connection -end -``` + payload = msg_payload_v1( + url, + payload_type; + id = string(uuid4()), + dataname = dataname, + transport = "link", + encoding = "none", + size = payload_size, + metadata = Dict{String, Any}() + ) + push!(payloads, payload) + end + end -**Example with pre-existing connection:** -```julia -using NATSBridge - -# Create connection once -conn = NATS.connect("nats://localhost:4222") - -# Send multiple messages using the same connection -for i in 1:100 - data = rand(1000) - smartsend( - "analysis_results", - [("table_data", data, "table")], - NATS_connection=conn, # Reuse connection - is_publish=true + # Create msg_envelope_v1 with all payloads + # Note: First positional argument is "send_to" (the NATS subject), not "subject" + env = msg_envelope_v1( + subject, # send_to: NATS subject to publish to + payloads; + correlation_id = correlation_id, + msg_id = msg_id, + msg_purpose = msg_purpose, + sender_name = sender_name, + sender_id = sender_id, + receiver_name = receiver_name, + receiver_id = receiver_id, + reply_to = reply_to, + reply_to_msg_id = reply_to_msg_id, + broker_url = broker_url, + metadata = Dict{String, Any}(), ) -end - -# Close connection when done -NATS.close(conn) -``` - -#### publish_message Function - -The `publish_message` function provides two overloads for publishing messages to NATS: - -**Overload 1 - URL-based publishing (creates new connection):** -```julia -function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) - conn = NATS.connect(broker_url) # Create NATS connection - publish_message(conn, subject, message, correlation_id) + + env_json_str = envelope_to_json(env) + + if is_publish == false + # skip publish + elseif is_publish == true && NATS_connection === nothing + publish_message(broker_url, subject, env_json_str, correlation_id) + elseif is_publish == true && NATS_connection !== nothing + publish_message(NATS_connection, subject, env_json_str, correlation_id) + end + + return (env, env_json_str) end ``` -**Overload 2 - Connection-based publishing (uses pre-existing connection):** +#### smartreceive Implementation + ```julia -function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) - try - NATS.publish(conn, subject, message) # Publish message to NATS - log_trace(correlation_id, "Message published to $subject") - finally - NATS.drain(conn) # Ensure connection is closed properly - end +function smartreceive( + msg::NATS.Msg; + fileserver_download_handler::Function = _fetch_with_backoff, + max_retries::Int = 5, + base_delay::Int = 100, + max_delay::Int = 5000 +)::JSON.Object{String, Any} + # Parse the JSON envelope + env_json_obj = JSON.parse(String(msg.payload)) + log_trace(env_json_obj["correlation_id"], "Processing received message") + + # Process all payloads in the envelope + payloads_list = Tuple{String, Any, String}[] + + num_payloads = length(env_json_obj["payloads"]) + + for i in 1:num_payloads + payload = env_json_obj["payloads"][i] + transport = String(payload["transport"]) + dataname = String(payload["dataname"]) + + if transport == "direct" + log_trace(env_json_obj["correlation_id"], "Direct transport - decoding payload '$dataname'") + + # Extract base64 payload from the payload + payload_b64 = String(payload["data"]) + + # Decode Base64 payload + payload_bytes = Base64.base64decode(payload_b64) + + # Deserialize based on type + data_type = String(payload["payload_type"]) + data = _deserialize_data(payload_bytes, data_type, env_json_obj["correlation_id"]) + + push!(payloads_list, (dataname, data, data_type)) + elseif transport == "link" + # Extract download URL from the payload + url = String(payload["data"]) + log_trace(env_json_obj["correlation_id"], "Link transport - fetching '$dataname' from URL: $url") + + # Fetch with exponential backoff using the download handler + downloaded_data = fileserver_download_handler(url, max_retries, base_delay, max_delay, env_json_obj["correlation_id"]) + + # Deserialize based on type + data_type = String(payload["payload_type"]) + data = _deserialize_data(downloaded_data, data_type, env_json_obj["correlation_id"]) + + push!(payloads_list, (dataname, data, data_type)) + else + error("Unknown transport type for payload '$dataname': $(transport)") + end + end + env_json_obj["payloads"] = payloads_list + return env_json_obj end ``` -**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. +#### _serialize_data Implementation -**Integration with smartsend:** ```julia -# When NATS_connection is provided to smartsend, it uses the connection-based publish_message -env, env_json_str = smartsend( - "my.subject", - [("data", payload_data, "type")], - NATS_connection=my_connection, # Pre-existing connection - is_publish=true -) -# Uses: publish_message(NATS_connection, subject, env_json_str, cid) - -# When NATS_connection is not provided, it uses the URL-based publish_message -env, env_json_str = smartsend( - "my.subject", - [("data", payload_data, "type")], - broker_url="nats://localhost:4222", - is_publish=true -) -# Uses: publish_message(broker_url, subject, env_json_str, cid) -``` - -**API Consistency Note:** -- **Julia:** Uses `NATS_connection` keyword parameter with function overloading for automatic connection management - -### Scenario 3: Live Binary Processing - -**Julia (Sender/Receiver):** -```julia -using NATSBridge - -# Binary data wrapped in list with type -smartsend( - "binary_input", - [("audio_chunk", binary_buffer, "binary")], - broker_url="nats://localhost:4222", - metadata=["sample_rate" => 44100, "channels" => 1] -) -``` - -### Scenario 4: Catch-Up (JetStream) - -**Julia (Producer/Consumer):** -```julia -using NATSBridge - -function publish_health_status(broker_url) - # Send status wrapped in list with type - status = Dict("cpu" => rand(), "memory" => rand()) - env, env_json_str = smartsend( - "health", - [("status", status, "dictionary")], - broker_url=broker_url - ) - sleep(5) # Every 5 seconds +function _serialize_data(data::Any, payload_type::String) + if payload_type == "text" + if isa(data, String) + data_bytes = Vector{UInt8}(data) + return data_bytes + else + error("Text data must be a String") + end + elseif payload_type == "dictionary" + json_str = JSON.json(data) + json_str_bytes = Vector{UInt8}(json_str) + return json_str_bytes + elseif payload_type == "arrowtable" + # Serialize DataFrame to Arrow IPC format + io = IOBuffer() + Arrow.write(io, data) + return take!(io) + elseif payload_type == "jsontable" + # Serialize to JSON + # data is Vector{NamedTuple} or Vector{Dict} + json_str = JSON.json(data) + return Vector{UInt8}(json_str) + elseif payload_type == "image" + if isa(data, Vector{UInt8}) + return data + else + error("Image data must be Vector{UInt8}") + end + elseif payload_type == "audio" + if isa(data, Vector{UInt8}) + return data + else + error("Audio data must be Vector{UInt8}") + end + elseif payload_type == "video" + if isa(data, Vector{UInt8}) + return data + else + error("Video data must be Vector{UInt8}") + end + elseif payload_type == "binary" + if isa(data, IOBuffer) + return take!(data) + elseif isa(data, Vector{UInt8}) + return data + else + error("Binary data must be binary (Vector{UInt8} or IOBuffer)") + end + else + error("Unknown payload_type: $payload_type") + end end ``` -### Scenario 5: Selection (Low Bandwidth) +#### _deserialize_data Implementation -**Focus:** Small Arrow tables. The Action: Julia wants to send a small DataFrame to show on a receiving application for the user to choose. - -**Julia (Sender/Receiver):** ```julia -using NATSBridge -using DataFrames - -# Create small DataFrame (e.g., 50KB - 500KB) -options_df = DataFrame( - id = 1:10, - name = ["Option A", "Option B", "Option C", "Option D", "Option E", - "Option F", "Option G", "Option H", "Option I", "Option J"], - description = ["Description A", "Description B", "Description C", "Description D", "Description E", - "Description F", "Description G", "Description H", "Description I", "Description J"] +function _deserialize_data( + data::Vector{UInt8}, + payload_type::String, + correlation_id::String ) - -# Convert to Arrow IPC stream -# Check payload size (< 1MB threshold) -# Publish directly to NATS with Base64-encoded payload -# Include metadata for dashboard selection context -env, env_json_str = smartsend( - "dashboard.selection", - [("options_table", options_df, "table")], - broker_url="nats://localhost:4222", - metadata=Dict("context" => "user_selection") -) -# env: msg_envelope_v1 with all metadata and payloads -# env_json_str: JSON string for publishing + if payload_type == "text" + return String(data) + elseif payload_type == "dictionary" + json_str = String(data) + return JSON.parse(json_str) + elseif payload_type == "arrowtable" + # Deserialize from Arrow IPC format + io = IOBuffer(data) + arrow_table = Arrow.Table(io) + return arrow_table + elseif payload_type == "jsontable" + # Deserialize from JSON format + # Returns Vector{NamedTuple} or Vector{Dict} + json_str = String(data) + parsed = JSON.parse(json_str) + return parsed + elseif payload_type == "image" + return data + elseif payload_type == "audio" + return data + elseif payload_type == "video" + return data + elseif payload_type == "binary" + return data + else + error("Unknown payload_type: $payload_type") + end +end ``` -**Use Case:** Julia server generates a list of available options (e.g., file selections, configuration presets) as a small DataFrame and sends to a receiving application for user selection. The selection is then sent back to Julia for processing. +#### _fetch_with_backoff Implementation -### Scenario 6: Chat System - -**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging. - -**Multi-Payload Support:** The system supports mixed-payload messages where a single message can contain multiple payloads with different transport strategies. The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. - -**Julia (Sender/Receiver):** ```julia -using NATSBridge - -# Build chat message with mixed payloads: -# - Text: direct transport (Base64) -# - Small images: direct transport (Base64) -# - Large images: link transport (HTTP URL) -# - Audio/video: link transport (HTTP URL) -# - Tables: direct or link depending on size -# - Files: link transport (HTTP URL) -# -# Each payload uses appropriate transport strategy: -# - Size < 1MB → direct (NATS + Base64) -# - Size >= 1MB → link (HTTP upload + NATS URL) -# -# Include claim-check metadata for delivery tracking -# Support bidirectional messaging with replyTo fields - -# Example: Chat with text, small image, and large file -chat_message = [ - ("message_text", "Hello, this is a test message!", "text"), - ("user_avatar", image_bytes, "image"), # Small image, direct transport - ("large_document", large_file_bytes, "binary") # Large file, link transport -] - -env, env_json_str = smartsend( - "chat.room123", - chat_message, - broker_url="nats://localhost:4222", - msg_purpose="chat", - reply_to="chat.room123.responses" +function _fetch_with_backoff( + url::String, + max_retries::Int, + base_delay::Int, + max_delay::Int, + correlation_id::String ) -# env: msg_envelope_v1 with all metadata and payloads -# env_json_str: JSON string for publishing + delay = base_delay + for attempt in 1:max_retries + try + response = HTTP.request("GET", url) + if response.status == 200 + log_trace(correlation_id, "Successfully fetched data from $url on attempt $attempt") + return response.body + else + error("Failed to fetch: $(response.status)") + end + catch e + log_trace(correlation_id, "Attempt $attempt failed: $(typeof(e))") + + if attempt < max_retries + sleep(delay / 1000.0) + delay = min(delay * 2, max_delay) + end + end + end + + error("Failed to fetch data after $max_retries attempts") +end ``` -**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components. +#### plik_oneshot_upload Implementation -**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads. +**Overload 1: Upload from binary data** + +```julia +function plik_oneshot_upload(file_server_url::String, dataname::String, data::Vector{UInt8}) + # Get upload id + url_getUploadID = "$file_server_url/upload" + headers = ["Content-Type" => "application/json"] + body = """{ "OneShot" : true }""" + http_response = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) + response_json = JSON.parse(http_response.body) + uploadid = response_json["id"] + uploadtoken = response_json["uploadToken"] + + # Upload file + file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream") + url_upload = "$file_server_url/file/$uploadid" + headers = ["X-UploadToken" => uploadtoken] + + form = HTTP.Form(Dict( + "file" => file_multipart + )) + + http_response = nothing + try + http_response = HTTP.post(url_upload, headers, form) + catch e + @error "Request failed" exception=e + end + response_json = JSON.parse(http_response.body) + fileid = response_json["id"] + + url = "$file_server_url/file/$uploadid/$fileid/$dataname" + + return Dict("status" => http_response.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) +end +``` + +**Overload 2: Upload from file path** + +```julia +function plik_oneshot_upload(file_server_url::String, filepath::String) + # Get upload id + filename = basename(filepath) + url_getUploadID = "$file_server_url/upload" + headers = ["Content-Type" => "application/json"] + body = """{ "OneShot" : true }""" + http_response = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) + response_json = JSON.parse(http_response.body) + + uploadid = response_json["id"] + uploadtoken = response_json["uploadToken"] + + # Upload file + url_upload = "$file_server_url/file/$uploadid" + headers = ["X-UploadToken" => uploadtoken] + http_response = open(filepath, "r") do file_stream + form = HTTP.Form(Dict("file" => file_stream)) + + # Adding status_exception=false prevents 4xx/5xx from triggering 'catch' + HTTP.post(url_upload, headers, form; status_exception = false) + end + + if !isnothing(http_response) && http_response.status == 200 + # Success - response already logged by caller + else + error("Failed to upload file: server returned status $(http_response.status)") + end + response_json = JSON.parse(http_response.body) + fileid = response_json["id"] + + # url of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" + url = "$file_server_url/file/$uploadid/$fileid/$filename" + + return Dict("status" => http_response.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) +end +``` + +--- + +### JavaScript Implementation + +#### Module Structure + +```javascript +// natsbridge.js +const nats = require('nats'); +const crypto = require('crypto'); +const fetch = require('node-fetch'); + +// UUID generation using built-in crypto module +const uuidv4 = () => crypto.randomUUID(); + +const DEFAULT_SIZE_THRESHOLD = 1_000_000; +const DEFAULT_BROKER_URL = 'nats://localhost:4222'; +const DEFAULT_FILESERVER_URL = 'http://localhost:8080'; + +class NATSClient { + constructor(url) { + this.url = url; + this.connection = null; + } + + async connect() { + this.connection = await nats.connect({ servers: this.url }); + return this.connection; + } + + async publish(subject, message) { + if (!this.connection) { + await this.connect(); + } + await this.connection.publish(subject, message); + } + + async close() { + if (this.connection) { + this.connection.close(); + } + } +} + +async function smartsend(subject, data, options = {}) { + // Implementation +} + +async function smartreceive(msg, options = {}) { + // Implementation +} + +module.exports = { + NATSClient, + smartsend, + smartreceive, + plikOneshotUpload, + fetchWithBackoff +}; +``` + +#### smartsend Implementation + +```javascript +const nats = require('nats'); +const crypto = require('crypto'); +const fetch = require('node-fetch'); +const arrow = require('apache-arrow'); + +// UUID generation using built-in crypto module +const uuidv4 = () => crypto.randomUUID(); + +const DEFAULT_SIZE_THRESHOLD = 1_000_000; +const DEFAULT_BROKER_URL = 'nats://localhost:4222'; +const DEFAULT_FILESERVER_URL = 'http://localhost:8080'; + +async function smartsend(subject, data, options = {}) { + const { + broker_url = DEFAULT_BROKER_URL, + fileserver_url = DEFAULT_FILESERVER_URL, + fileserver_upload_handler = plikOneshotUpload, + size_threshold = DEFAULT_SIZE_THRESHOLD, + correlation_id = uuidv4(), + msg_purpose = 'chat', + sender_name = 'NATSBridge', + receiver_name = '', + receiver_id = '', + reply_to = '', + reply_to_msg_id = '', + is_publish = true, + nats_connection = null, + msg_id = uuidv4(), + sender_id = uuidv4() + } = options; + + console.log(`[Correlation: ${correlation_id}] Starting smartsend for subject: ${subject}`); + + // Process payloads + const payloads = []; + for (const [dataname, payloadData, payloadType] of data) { + const payloadBytes = await serializeData(payloadData, payloadType); + const payloadSize = payloadBytes.byteLength; + + console.log(`[Correlation: ${correlation_id}] Serialized payload '${dataname}' (type: ${payloadType}) size: ${payloadSize} bytes`); + + if (payloadSize < size_threshold) { + // Direct path + const payloadB64 = bufferToBase64(payloadBytes); + console.log(`[Correlation: ${correlation_id}] Using direct transport for ${payloadSize} bytes`); + + payloads.push({ + id: uuidv4(), + dataname, + payload_type: payloadType, + transport: 'direct', + encoding: 'base64', + size: payloadSize, + data: payloadB64, + metadata: { payload_bytes: payloadSize } + }); + } else { + // Link path + console.log(`[Correlation: ${correlation_id}] Using link transport, uploading to fileserver`); + + const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes); + + if (response.status !== 200) { + throw new Error(`Failed to upload data to fileserver: ${response.status}`); + } + + console.log(`[Correlation: ${correlation_id}] Uploaded to URL: ${response.url}`); + + payloads.push({ + id: uuidv4(), + dataname, + payload_type: payloadType, + transport: 'link', + encoding: 'none', + size: payloadSize, + data: response.url, + metadata: {} + }); + } + } + + // Build envelope + const env = { + correlation_id, + msg_id, + timestamp: new Date().toISOString(), + send_to: subject, + msg_purpose, + sender_name, + sender_id, + receiver_name, + receiver_id, + reply_to, + reply_to_msg_id, + broker_url, + metadata: {}, + payloads + }; + + const env_json_str = JSON.stringify(env); + + if (is_publish) { + if (nats_connection) { + await publishMessage(nats_connection, subject, env_json_str, correlation_id); + } else { + await publishMessage(broker_url, subject, env_json_str, correlation_id); + } + } + + return [env, env_json_str]; +} +``` + +#### serializeData Implementation + +```javascript +const arrow = require('apache-arrow'); + +async function serializeData(data, payload_type) { + if (payload_type === 'text') { + if (typeof data === 'string') { + return Buffer.from(data, 'utf8'); + } else { + throw new Error('Text data must be a string'); + } + } else if (payload_type === 'dictionary') { + const jsonStr = JSON.stringify(data); + return Buffer.from(jsonStr, 'utf8'); + } else if (payload_type === 'arrowtable') { + // Convert Array to Arrow IPC + if (!Array.isArray(data) || data.length === 0) { + throw new Error('arrowtable data must be a non-empty array of objects'); + } + + // Create schema from first row + const schemaFields = Object.keys(data[0]).map(key => + new arrow.Field(key, arrow.any()) + ); + const schema = new arrow.Schema(schemaFields); + + // Create writer + const writer = new arrow.RecordBatchWriter([schema]); + + // Write rows + for (const row of data) { + const recordBatch = arrow.recordBatch.fromObjects([row], schema); + writer.write(recordBatch); + } + await writer.close(); + + // Read buffer + return writer.toBuffer(); + } else if (payload_type === 'jsontable') { + // Serialize directly to JSON + const jsonStr = JSON.stringify(data); + return Buffer.from(jsonStr, 'utf8'); + } else if (payload_type === 'image') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Image data must be Uint8Array or Buffer'); + } + } else if (payload_type === 'audio') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Audio data must be Uint8Array or Buffer'); + } + } else if (payload_type === 'video') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Video data must be Uint8Array or Buffer'); + } + } else if (payload_type === 'binary') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Binary data must be Uint8Array or Buffer'); + } + } else { + throw new Error(`Unknown payload_type: ${payload_type}`); + } +} + +function bufferToBase64(buffer) { + return buffer.toString('base64'); +} +``` + +#### deserializeData Implementation + +```javascript +const arrow = require('apache-arrow'); + +async function deserializeData(data, payload_type, correlation_id) { + if (payload_type === 'text') { + return Buffer.from(data).toString('utf8'); + } else if (payload_type === 'dictionary') { + const jsonStr = Buffer.from(data).toString('utf8'); + return JSON.parse(jsonStr); + } else if (payload_type === 'arrowtable') { + // Deserialize from Arrow IPC + const buffer = Buffer.from(data); + const table = arrow.tableFromRawBytes(buffer); + return table; + } else if (payload_type === 'jsontable') { + // Deserialize from JSON - returns Array + const jsonStr = Buffer.from(data).toString('utf8'); + return JSON.parse(jsonStr); + } else if (payload_type === 'image') { + return Buffer.from(data); + } else if (payload_type === 'audio') { + return Buffer.from(data); + } else if (payload_type === 'video') { + return Buffer.from(data); + } else if (payload_type === 'binary') { + return Buffer.from(data); + } else { + throw new Error(`Unknown payload_type: ${payload_type}`); + } +} +``` + +#### fetchWithBackoff Implementation + +```javascript +async function fetchWithBackoff(url, max_retries, base_delay, max_delay, correlation_id) { + let delay = base_delay; + + for (let attempt = 1; attempt <= max_retries; attempt++) { + try { + const response = await fetch(url); + + if (response.status === 200) { + console.log(`[Correlation: ${correlation_id}] Successfully fetched data from ${url} on attempt ${attempt}`); + return await response.arrayBuffer(); + } else { + throw new Error(`Failed to fetch: ${response.status}`); + } + } catch (e) { + console.log(`[Correlation: ${correlation_id}] Attempt ${attempt} failed: ${e.constructor.name}`); + + if (attempt < max_retries) { + await new Promise(resolve => setTimeout(resolve, delay)); + delay = Math.min(delay * 2, max_delay); + } + } + } + + throw new Error(`Failed to fetch data after ${max_retries} attempts`); +} +``` + +#### plikOneshotUpload Implementation + +```javascript +async function plikOneshotUpload(file_server_url, dataname, data) { + // Get upload id + const url_getUploadID = `${file_server_url}/upload`; + const headers = { 'Content-Type': 'application/json' }; + const body = JSON.stringify({ OneShot: true }); + + const http_response = await fetch(url_getUploadID, { + method: 'POST', + headers, + body + }); + + const response_json = await http_response.json(); + const uploadid = response_json.id; + const uploadtoken = response_json.uploadToken; + + // Upload file + const url_upload = `${file_server_url}/file/${uploadid}`; + const form = new FormData(); + const blob = new Blob([data]); + form.append('file', blob, dataname); + + const upload_headers = { + 'X-UploadToken': uploadtoken + }; + + const upload_response = await fetch(url_upload, { + method: 'POST', + headers: upload_headers, + body: form + }); + + const upload_json = await upload_response.json(); + const fileid = upload_json.id; + + const url = `${file_server_url}/file/${uploadid}/${fileid}/${dataname}`; + + return { + status: upload_response.status, + uploadid, + fileid, + url + }; +} +``` + +--- + +### Python Implementation + +#### Module Structure + +```python +# natsbridge.py +import asyncio +import base64 +import json +import uuid +import time +from typing import Any, Dict, List, Tuple, Union, Callable +from dataclasses import dataclass, field +from datetime import datetime + +try: + import pyarrow as arrow + import pyarrow.parquet as pq + ARROW_AVAILABLE = True +except ImportError: + ARROW_AVAILABLE = False + +try: + import aiohttp + import nats + from nats.aio.client import Client as NATSClient + NATS_AVAILABLE = True +except ImportError: + NATS_AVAILABLE = False + + +DEFAULT_SIZE_THRESHOLD = 1_000_000 +DEFAULT_BROKER_URL = "nats://localhost:4222" +DEFAULT_FILESERVER_URL = "http://localhost:8080" + + +@dataclass +class MsgPayloadV1: + """Message payload structure.""" + id: str + dataname: str + payload_type: str + transport: str + encoding: str + size: int + data: Union[str, bytes] + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class MsgEnvelopeV1: + """Message envelope structure.""" + correlation_id: str + msg_id: str + timestamp: str + send_to: str + msg_purpose: str + sender_name: str + sender_id: str + receiver_name: str + receiver_id: str + reply_to: str + reply_to_msg_id: str + broker_url: str + metadata: Dict[str, Any] = field(default_factory=dict) + payloads: List[MsgPayloadV1] = field(default_factory=list) + + +class NATSBridge: + """Cross-platform NATS bridge implementation.""" + + def __init__(self, broker_url: str = None, fileserver_url: str = None): + self.broker_url = broker_url or DEFAULT_BROKER_URL + self.fileserver_url = fileserver_url or DEFAULT_FILESERVER_URL + self._nats_client: NATSClient = None + + async def smartsend(self, subject: str, data: List[Tuple[str, Any, str]], **kwargs) -> Tuple[Dict, str]: + """Send data via NATS.""" + pass + + async def smartreceive(self, msg: Any, **kwargs) -> Dict: + """Receive and process NATS message.""" + pass +``` + +#### smartsend Implementation + +```python +import asyncio +import base64 +import json +import uuid +from typing import Any, Dict, List, Tuple, Union, Callable +from datetime import datetime + +DEFAULT_SIZE_THRESHOLD = 1_000_000 +DEFAULT_BROKER_URL = "nats://localhost:4222" +DEFAULT_FILESERVER_URL = "http://localhost:8080" + + +async def smartsend( + subject: str, + data: List[Tuple[str, Any, str]], + broker_url: str = DEFAULT_BROKER_URL, + fileserver_url: str = DEFAULT_FILESERVER_URL, + fileserver_upload_handler: Callable = plik_oneshot_upload, + size_threshold: int = DEFAULT_SIZE_THRESHOLD, + correlation_id: str = None, + msg_purpose: str = "chat", + sender_name: str = "NATSBridge", + receiver_name: str = "", + receiver_id: str = "", + reply_to: str = "", + reply_to_msg_id: str = "", + is_publish: bool = True, + nats_connection: Any = None, + msg_id: str = None, + sender_id: str = None +) -> Tuple[Dict, str]: + """ + Send data via NATS with automatic transport selection. + + Args: + subject: NATS subject to publish to + data: List of (dataname, data, type) tuples + **kwargs: Additional options + + Returns: + Tuple of (env, env_json_str) + """ + if correlation_id is None: + correlation_id = str(uuid.uuid4()) + if msg_id is None: + msg_id = str(uuid.uuid4()) + if sender_id is None: + sender_id = str(uuid.uuid4()) + + print(f"[Correlation: {correlation_id}] Starting smartsend for subject: {subject}") + + # Process payloads + payloads = [] + for dataname, payload_data, payload_type in data: + payload_bytes = _serialize_data(payload_data, payload_type) + payload_size = len(payload_bytes) + + print(f"[Correlation: {correlation_id}] Serialized payload '{dataname}' (type: {payload_type}) size: {payload_size} bytes") + + if payload_size < size_threshold: + # Direct path + payload_b64 = base64.b64encode(payload_bytes).decode('utf-8') + print(f"[Correlation: {correlation_id}] Using direct transport for {payload_size} bytes") + + payloads.append({ + 'id': str(uuid.uuid4()), + 'dataname': dataname, + 'payload_type': payload_type, + 'transport': 'direct', + 'encoding': 'base64', + 'size': payload_size, + 'data': payload_b64, + 'metadata': {'payload_bytes': payload_size} + }) + else: + # Link path + print(f"[Correlation: {correlation_id}] Using link transport, uploading to fileserver") + + response = await fileserver_upload_handler(fileserver_url, dataname, payload_bytes) + + if response['status'] != 200: + raise Exception(f"Failed to upload data to fileserver: {response['status']}") + + print(f"[Correlation: {correlation_id}] Uploaded to URL: {response['url']}") + + payloads.append({ + 'id': str(uuid.uuid4()), + 'dataname': dataname, + 'payload_type': payload_type, + 'transport': 'link', + 'encoding': 'none', + 'size': payload_size, + 'data': response['url'], + 'metadata': {} + }) + + # Build envelope + env = { + 'correlation_id': correlation_id, + 'msg_id': msg_id, + 'timestamp': datetime.utcnow().isoformat() + 'Z', + 'send_to': subject, + 'msg_purpose': msg_purpose, + 'sender_name': sender_name, + 'sender_id': sender_id, + 'receiver_name': receiver_name, + 'receiver_id': receiver_id, + 'reply_to': reply_to, + 'reply_to_msg_id': reply_to_msg_id, + 'broker_url': broker_url, + 'metadata': {}, + 'payloads': payloads + } + + env_json_str = json.dumps(env) + + if is_publish: + if nats_connection: + await publish_message(nats_connection, subject, env_json_str, correlation_id) + else: + await publish_message(broker_url, subject, env_json_str, correlation_id) + + return env, env_json_str +``` + +#### serializeData Implementation + +```python +import base64 +import json +from typing import Any + +try: + import pyarrow as arrow + import pyarrow.feather as feather + import pyarrow.ipc as ipc + ARROW_AVAILABLE = True +except ImportError: + ARROW_AVAILABLE = False + + +def _serialize_data(data: Any, payload_type: str) -> bytes: + """ + Serialize data to bytes based on type. + + Note: Python uses "table" as a single type for both Arrow and JSON table + serialization. Julia/JavaScript use separate "arrowtable" and "jsontable" types. + """ + if payload_type == 'text': + if isinstance(data, str): + return data.encode('utf-8') + else: + raise ValueError('Text data must be a string') + elif payload_type == 'dictionary': + json_str = json.dumps(data) + return json_str.encode('utf-8') + elif payload_type == 'table': + # Python uses "table" for both arrowtable and jsontable + if not ARROW_AVAILABLE: + raise RuntimeError('pyarrow not available for table serialization') + + import io + buf = io.BytesIO() + import pandas as pd + if isinstance(data, pd.DataFrame): + # Serialize DataFrame to Arrow + table = arrow.Table.from_pandas(data) + sink = ipc.new_file(buf, table.schema) + ipc.write_table(table, sink) + sink.close() + return buf.getvalue() + elif isinstance(data, arrow.Table): + sink = ipc.new_file(buf, data.schema) + ipc.write_table(data, sink) + sink.close() + return buf.getvalue() + else: + raise ValueError('Table data must be a pandas DataFrame or pyarrow Table') + elif payload_type in ('image', 'audio', 'video', 'binary'): + if isinstance(data, (bytes, bytearray)): + return bytes(data) + else: + raise ValueError(f'{payload_type} data must be bytes') + else: + raise ValueError(f'Unknown payload_type: {payload_type}') +``` + +#### deserializeData Implementation + +```python +import base64 +import json +from typing import Any + +try: + import pyarrow as arrow + import pyarrow.feather as feather + import pyarrow.ipc as ipc + ARROW_AVAILABLE = True +except ImportError: + ARROW_AVAILABLE = False + + +def _deserialize_data(data: bytes, payload_type: str, correlation_id: str) -> Any: + """ + Deserialize bytes to data based on type. + + Note: Python uses "table" as a single type for both Arrow and JSON table + deserialization. Julia/JavaScript use separate "arrowtable" and "jsontable" types. + """ + if payload_type == 'text': + return data.decode('utf-8') + elif payload_type == 'dictionary': + json_str = data.decode('utf-8') + return json.loads(json_str) + elif payload_type == 'table': + # Python uses "table" for both arrowtable and jsontable + if not ARROW_AVAILABLE: + raise RuntimeError('pyarrow not available for table deserialization') + + import io + buf = io.BytesIO(data) + reader = ipc.open_file(buf) + return reader.read_all().to_pandas() + elif payload_type in ('image', 'audio', 'video', 'binary'): + return data + else: + raise ValueError(f'Unknown payload_type: {payload_type}') +``` + +#### fetchWithBackoff Implementation + +```python +import asyncio +import aiohttp +from typing import Callable + + +async def fetch_with_backoff( + url: str, + max_retries: int, + base_delay: int, + max_delay: int, + correlation_id: str +) -> bytes: + """Fetch URL with exponential backoff.""" + delay = base_delay + + for attempt in range(1, max_retries + 1): + try: + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + if response.status == 200: + print(f"[Correlation: {correlation_id}] Successfully fetched data from {url} on attempt {attempt}") + return await response.read() + else: + raise Exception(f"Failed to fetch: {response.status}") + except Exception as e: + print(f"[Correlation: {correlation_id}] Attempt {attempt} failed: {type(e).__name__}") + + if attempt < max_retries: + await asyncio.sleep(delay / 1000.0) + delay = min(delay * 2, max_delay) + + raise Exception(f"Failed to fetch data after {max_retries} attempts") +``` + +#### plikOneshotUpload Implementation + +```python +import aiohttp +import json +from typing import Dict, Any + + +async def plik_oneshot_upload( + file_server_url: str, + dataname: str, + data: bytes +) -> Dict[str, Any]: + """Upload data to plik server in one-shot mode.""" + + # Get upload id + async with aiohttp.ClientSession() as session: + url_getUploadID = f"{file_server_url}/upload" + headers = {'Content-Type': 'application/json'} + body = json.dumps({"OneShot": True}) + + async with session.post(url_getUploadID, headers=headers, data=body) as response: + response_json = await response.json() + uploadid = response_json['id'] + uploadtoken = response_json['uploadToken'] + + # Upload file + url_upload = f"{file_server_url}/file/{uploadid}" + headers = {'X-UploadToken': uploadtoken} + + form = aiohttp.FormData() + form.add_field('file', data, filename=dataname, content_type='application/octet-stream') + + async with session.post(url_upload, headers=headers, data=form) as upload_response: + upload_json = await upload_response.json() + fileid = upload_json['id'] + + url = f"{file_server_url}/file/{uploadid}/{fileid}/{dataname}" + + return { + 'status': upload_response.status, + 'uploadid': uploadid, + 'fileid': fileid, + 'url': url + } +``` + +--- + +### MicroPython Implementation + +#### Limitations + +MicroPython has significant constraints compared to desktop implementations: + +| Feature | Desktop | MicroPython | +|---------|---------|-------------| +| Memory | Unlimited | ~256KB - 1MB | +| Arrow IPC | ✅ | ❌ (not supported) | +| Async/Await | ✅ | ❌ (synchronous only) | +| Large payloads (>1MB) | ✅ | ❌ (enforced limit) | +| arrowtable | ✅ | ❌ (not supported) | +| jsontable | ✅ | ❌ (not supported) | +| Multiple payloads | ✅ | ⚠️ (limited) | + +**Note:** MicroPython does NOT support table types (`arrowtable` or `jsontable`) due to memory constraints. + +#### Module Structure + +```python +# natsbridge_mpy.py (MicroPython) +import network +import time +import json +import base64 +import uos +import struct +import random + +# Constants +DEFAULT_SIZE_THRESHOLD = 100000 # 100KB for MicroPython +DEFAULT_BROKER_URL = "nats://localhost:4222" +DEFAULT_FILESERVER_URL = "http://localhost:8080" +MAX_PAYLOAD_SIZE = 50000 # Hard limit (lower than threshold for safety) + +# Note: MicroPython does NOT support table types (arrowtable/jsontable) +# Only supports: text, dictionary, image, audio, video, binary + + +class NATSBridge: + """MicroPython NATS bridge implementation.""" + + def __init__(self, broker_url=None, fileserver_url=None): + self.broker_url = broker_url or DEFAULT_BROKER_URL + self.fileserver_url = fileserver_url or DEFAULT_FILESERVER_URL + self._nats_conn = None + + def smartsend(self, subject, data, **kwargs): + """Send data (synchronous).""" + correlation_id = self._generate_uuid() + msg_id = self._generate_uuid() + sender_id = self._generate_uuid() + + print(f"[Correlation: {correlation_id}] Starting smartsend") + + payloads = [] + for dataname, payload_data, payload_type in data: + payload_bytes = self._serialize_data(payload_data, payload_type) + payload_size = len(payload_bytes) + + if payload_size > MAX_PAYLOAD_SIZE: + raise MemoryError(f"Payload {dataname} exceeds max size {MAX_PAYLOAD_SIZE}") + + if payload_size < DEFAULT_SIZE_THRESHOLD: + # Direct path + payload_b64 = base64.b64encode(payload_bytes).decode('ascii') + payloads.append({ + 'id': self._generate_uuid(), + 'dataname': dataname, + 'payload_type': payload_type, + 'transport': 'direct', + 'encoding': 'base64', + 'size': payload_size, + 'data': payload_b64 + }) + else: + # Link path (limited support) + response = self._sync_fileserver_upload(self.fileserver_url, dataname, payload_bytes) + payloads.append({ + 'id': self._generate_uuid(), + 'dataname': dataname, + 'payload_type': payload_type, + 'transport': 'link', + 'encoding': 'none', + 'size': payload_size, + 'data': response['url'] + }) + + env = { + 'correlation_id': correlation_id, + 'msg_id': msg_id, + 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime()), + 'send_to': subject, + 'msg_purpose': kwargs.get('msg_purpose', 'chat'), + 'sender_name': kwargs.get('sender_name', 'NATSBridge'), + 'sender_id': sender_id, + 'receiver_name': kwargs.get('receiver_name', ''), + 'receiver_id': kwargs.get('receiver_id', ''), + 'reply_to': kwargs.get('reply_to', ''), + 'reply_to_msg_id': kwargs.get('reply_to_msg_id', ''), + 'broker_url': self.broker_url, + 'metadata': {}, + 'payloads': payloads + } + + env_json_str = json.dumps(env) + + # Publish + self._publish(subject, env_json_str, correlation_id) + + return env, env_json_str + + def smartreceive(self, msg, **kwargs): + """Receive and process message (synchronous).""" + env_json_obj = json.loads(msg.payload) + correlation_id = env_json_obj['correlation_id'] + + payloads_list = [] + for payload in env_json_obj['payloads']: + transport = payload['transport'] + dataname = payload['dataname'] + + if transport == 'direct': + payload_b64 = payload['data'] + payload_bytes = base64.b64decode(payload_b64) + data_type = payload['payload_type'] + data = self._deserialize_data(payload_bytes, data_type) + payloads_list.append((dataname, data, data_type)) + elif transport == 'link': + url = payload['data'] + downloaded_data = self._sync_fileserver_download( + url, + kwargs.get('max_retries', 3), + kwargs.get('base_delay', 100), + kwargs.get('max_delay', 1000), + correlation_id + ) + data_type = payload['payload_type'] + data = self._deserialize_data(downloaded_data, data_type) + payloads_list.append((dataname, data, data_type)) + + env_json_obj['payloads'] = payloads_list + return env_json_obj + + def _serialize_data(self, data, payload_type): + """ + Serialize data (MicroPython version). + + Note: MicroPython does NOT support table types (arrowtable/jsontable). + Only supports: text, dictionary, image, audio, video, binary + """ + if payload_type == 'text': + if isinstance(data, str): + return data.encode('utf-8') + else: + raise ValueError('Text data must be a string') + elif payload_type == 'dictionary': + json_str = json.dumps(data) + return json_str.encode('utf-8') + elif payload_type in ('image', 'audio', 'video', 'binary'): + if isinstance(data, (bytes, bytearray, memoryview)): + return bytes(data) + else: + raise ValueError(f'{payload_type} data must be bytes') + else: + raise ValueError(f'Unknown payload_type: {payload_type}') + + def _deserialize_data(self, data, payload_type): + """ + Deserialize data (MicroPython version). + + Note: MicroPython does NOT support table types (arrowtable/jsontable). + Only supports: text, dictionary, image, audio, video, binary + """ + if payload_type == 'text': + return data.decode('utf-8') + elif payload_type == 'dictionary': + json_str = data.decode('utf-8') + return json.loads(json_str) + elif payload_type in ('image', 'audio', 'video', 'binary'): + return data + else: + raise ValueError(f'Unknown payload_type: {payload_type}') + + def _generate_uuid(self): + """Generate simple UUID (MicroPython compatible).""" + return 'mp-%04x%04x-%04x-%04x-%04x-%04x%04x%04x' % ( + time.time_ns() // (10**6) % 0xFFFFFFFF, + time.time_ns() % 0xFFFFFFFF, + time.time_ns() >> 32 & 0xFFFF, + time.time_ns() >> 48 & 0xFFFF, + time.time_ns() >> 64 & 0xFFFF, + time.time_ns() >> 80 & 0xFFFF, + time.time_ns() >> 96 & 0xFFFF, + time.time_ns() >> 112 & 0xFFFF + ) + + def _sync_fileserver_upload(self, url, dataname, data): + """Synchronous file upload (limited).""" + # Simplified implementation for MicroPython + # In practice, would use network.HTTP or similar + raise NotImplementedError("File upload not implemented in MicroPython") + + def _sync_fileserver_download(self, url, max_retries, base_delay, max_delay, correlation_id): + """Synchronous file download with backoff.""" + # Simplified implementation for MicroPython + raise NotImplementedError("File download not implemented in MicroPython") + + def _publish(self, subject, message, correlation_id): + """Publish message to NATS.""" + # Simplified implementation for MicroPython + raise NotImplementedError("NATS publishing not implemented in MicroPython") +``` + +--- ## Configuration @@ -548,99 +1714,105 @@ env, env_json_str = smartsend( | Variable | Default | Description | |----------|---------|-------------| | `NATS_URL` | `nats://localhost:4222` | NATS server URL | -| `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL (base URL without `/upload` suffix) | -| `SIZE_THRESHOLD` | `1_000_000` | Size threshold in bytes (1MB) | +| `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL | +| `SIZE_THRESHOLD` | `1000000` | Size threshold in bytes (1MB) | -### Message Envelope Schema +### MicroPython Configuration -```json -{ - "correlation_id": "uuid-v4-string", - "msg_id": "uuid-v4-string", - "timestamp": "2024-01-15T10:30:00Z", - - "send_to": "topic/subject", - "msg_purpose": "ACK | NACK | updateStatus | shutdown | chat", - "sender_name": "agent-wine-web-frontend", - "sender_id": "uuid4", - "receiver_name": "agent-backend", - "receiver_id": "uuid4", - "reply_to": "topic", - "reply_to_msg_id": "uuid4", - "broker_url": "nats://localhost:4222", - - "metadata": { - "content_type": "application/octet-stream", - "content_length": 123456 - }, - - "payloads": [ - { - "id": "uuid4", - "dataname": "login_image", - "payload_type": "image", - "transport": "direct", - "encoding": "base64", - "size": 15433, - "data": "base64-encoded-string", - "metadata": { - "checksum": "sha256_hash" - } - } - ] -} +```python +# micropython.conf +NATS_URL = "nats://broker.local:4222" +FILESERVER_URL = "http://fileserver.local:8080" +SIZE_THRESHOLD = 100000 # Lower threshold for memory-constrained devices +MAX_PAYLOAD_SIZE = 50000 # Hard limit for MicroPython ``` +--- + ## Performance Considerations ### Zero-Copy Reading -- Use Arrow's memory-mapped file reading -- Avoid unnecessary data copying during deserialization -- Use Apache Arrow's native IPC reader + +| Platform | Strategy | +|----------|----------| +| **Julia** | `Arrow.read()` with memory-mapped files | +| **JavaScript** | `ArrayBuffer` with `DataView` | +| **Python** | `pyarrow` memory mapping | +| **MicroPython** | Not available (streaming only) | ### Exponential Backoff -- Maximum retry count: 5 -- Base delay: 100ms, max delay: 5000ms + +All platforms implement exponential backoff for HTTP downloads: + +```python +# Python +async def fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id): + delay = base_delay + for attempt in range(1, max_retries + 1): + try: + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + if response.status == 200: + return await response.read() + except Exception as e: + if attempt < max_retries: + await asyncio.sleep(delay / 1000.0) + delay = min(delay * 2, max_delay) + raise Exception("Failed to fetch after max retries") +``` ### Correlation ID Logging -- Log correlation_id at every stage -- Include: send, receive, serialize, deserialize -- Use structured logging format + +All platforms use correlation IDs for distributed tracing: + +``` +[timestamp] [Correlation: abc123] Message published to subject +``` + +### Serialization Performance + +| Format | Use Case | Pros | Cons | +|--------|----------|------|------| +| `arrowtable` | Large tabular data | Fast, zero-copy, schema-preserving | Binary format, requires Arrow library, not supported in MicroPython | +| `jsontable` | Small/medium tabular data | Human-readable, universal support, works in MicroPython | Slower, larger size, no schema enforcement | + +--- ## Testing -Run the test scripts for Julia: +### Test File Organization -### Julia Tests +| Platform | Sender Tests | Receiver Tests | +|----------|--------------|----------------| +| **Julia** | `test/test_julia_*_sender.jl` | `test/test_julia_*_receiver.jl` | +| **JavaScript** | `test/test_js_*_sender.js` | `test/test_js_*_receiver.js` | +| **Python** | `test/test_py_*_sender.py` | `test/test_py_*_receiver.py` | + +### Run Tests ```bash -# Text message exchange -julia test/test_julia_to_julia_text_sender.jl -julia test/test_julia_to_julia_text_receiver.jl +# Julia +julia test/test_julia_text_sender.jl +julia test/test_julia_text_receiver.jl -# Dictionary exchange -julia test/test_julia_to_julia_dict_sender.jl -julia test/test_julia_to_julia_dict_receiver.jl +# JavaScript (Node.js) +node test/test_js_text_sender.js +node test/test_js_text_receiver.js -# File transfer -julia test/test_julia_to_julia_file_sender.jl -julia test/test_julia_to_julia_file_receiver.jl - -# Mixed payload types -julia test/test_julia_to_julia_mix_payloads_sender.jl -julia test/test_julia_to_julia_mix_payloads_receiver.jl - -# Table exchange -julia test/test_julia_to_julia_table_sender.jl -julia test/test_julia_to_julia_table_receiver.jl +# Python +python3 test/test_py_text_sender.py +python3 test/test_py_text_receiver.py ``` +--- + ## Troubleshooting ### Common Issues 1. **NATS Connection Failed** - Ensure NATS server is running + - Check `broker_url` configuration 2. **HTTP Upload Failed** - Ensure file server is running @@ -650,7 +1822,38 @@ julia test/test_julia_to_julia_table_receiver.jl 3. **Arrow IPC Deserialization Error** - Ensure data is properly serialized to Arrow format - Check Arrow version compatibility + - MicroPython doesn't support Arrow IPC -## License +4. **Memory Constraints (MicroPython)** + - Reduce `size_threshold` + - Use direct transport only (< 100KB) + - Avoid large payloads + - Use `jsontable` instead of `arrowtable` (arrowtable not supported) -MIT \ No newline at end of file +--- + +## Summary + +This cross-platform NATS bridge provides: + +1. **High-Level API Parity**: Identical `smartsend()` and `smartreceive()` signatures across all platforms +2. **Idiomatic Implementations**: + - **Julia**: Multiple dispatch, struct-based design, native Arrow IPC + - **JavaScript**: Async/await, prototype-based utilities, class-based NATS client + - **Python**: Class-based design with dataclasses, type hints, async/await + - **MicroPython**: Synchronous API, memory-constrained optimizations +3. **Message Format Consistency**: Identical JSON schemas across all platforms +4. **Handler Abstraction**: File server operations abstracted through configurable handlers +5. **Platform-Specific Optimizations**: + - **Arrow IPC** (`arrowtable`): Efficient binary format for large tabular data (not supported in MicroPython) + - **JSON** (`jsontable`): Universal human-readable format for smaller tables (works in Julia, JavaScript, Python; NOT supported in MicroPython) + +The Julia implementation in [`src/NATSBridge.jl`](src/NATSBridge.jl:1) serves as the ground truth for API design and behavior. + +### Datatype Summary + +| Datatype | Serialization | Use Case | Encoding | Supported Platforms | +|----------|---------------|----------|----------|---------------------| +| `arrowtable` | Apache Arrow IPC | Large tabular data, schema-preserving | `arrow-ipc` → `base64` | Julia, JavaScript, Python | +| `jsontable` | JSON | Small/medium tabular data, human-readable | `json` → `base64` | Julia, JavaScript, Python | +| `table` | Apache Arrow IPC (Python only) | Python's unified table type | `arrow-ipc` → `base64` | Python | diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 0000000..562e082 --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,741 @@ +# Cross-Platform NATSBridge Tutorial + +A step-by-step guide to get started with NATSBridge across **Julia**, **JavaScript**, and **Python/MicroPython**. + +## Table of Contents + +1. [Overview](#overview) +2. [Prerequisites](#prerequisites) +3. [Installation](#installation) +4. [Quick Start](#quick-start) +5. [Basic Examples](#basic-examples) +6. [Advanced Usage](#advanced-usage) + +--- + +## Overview + +NATSBridge enables seamless communication across platforms through NATS, with automatic transport selection based on payload size: + +- **Direct Transport**: Payloads < 1MB are sent directly via NATS (Base64 encoded) +- **Link Transport**: Payloads >= 1MB are uploaded to an HTTP file server and referenced via URL + +### Cross-Platform API Parity + +All three platforms use the same high-level API: + +``` +# Input format +smartsend(subject, [(dataname, data, type), ...], options) + +# Output format +(env, env_json_str) = smartsend(...) +env = smartreceive(msg, options) +``` + +**Important Platform Differences:** + +1. **Encoding field:** Julia and JavaScript preserve the original serialization format in the encoding field (`"base64"`, `"json"`, or `"arrow-ipc"`), while Python and MicroPython always use `"base64"` for all direct transport payloads. + +2. **Async vs Sync:** JavaScript and Python desktop use async/await, while MicroPython uses synchronous API. + +### Supported Payload Types + +| Type | Julia | JavaScript | Python | MicroPython | +|------|-------|------------|--------|-------------| +| `text` | `String` | `string` | `str` | `str` | +| `dictionary` | `Dict` | `Object` | `dict` | `dict` | +| `arrowtable` | `DataFrame` | `Array` | `pandas.DataFrame` | ❌ | +| `jsontable` | `Vector{NamedTuple}` | `Array` | `list[dict]` | ❌ | +| `table` | ❌ | ❌ | `pandas.DataFrame` | ❌ | +| `image` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` | +| `audio` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` | +| `video` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` | +| `binary` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` | + +**Note on MicroPython:** MicroPython does not support table types (`arrowtable`, `jsontable`, or `table`) due to memory constraints. Use `dictionary` or `binary` instead. + +--- + +## Prerequisites + +Before you begin, ensure you have: + +1. **NATS Server** running (or accessible) +2. **HTTP File Server** (optional, for large payloads > 1MB) +3. **Platform-specific packages** installed + +--- + +## Installation + +### Julia + +```julia +using Pkg +Pkg.add("NATS") +Pkg.add("Arrow") +Pkg.add("JSON3") +Pkg.add("HTTP") +Pkg.add("UUIDs") +Pkg.add("Dates") +``` + +### JavaScript (Node.js) + +```bash +npm install nats uuid apache-arrow node-fetch +``` + +### JavaScript (Browser) + +```html + + +``` + +### Python (Desktop) + +```bash +pip install nats-py aiohttp pyarrow pandas +``` + +### MicroPython + +Uses built-in modules: `network`, `socket`, `time`, `json`, `base64` + +--- + +## Quick Start + +### Step 1: Start NATS Server + +```bash +docker run -p 4222:4222 nats:latest +``` + +### Step 2: Start HTTP File Server (Optional) + +```bash +mkdir -p /tmp/fileserver +python3 -m http.server 8080 --directory /tmp/fileserver +``` + +### Step 3: Send Your First Message + +#### Julia + +```julia +using NATSBridge + +# Send a text message +data = [("message", "Hello World", "text")] +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") +# env: msg_envelope_v1 struct with all metadata and payloads +# env_json_str: JSON string representation of the envelope for publishing +println("Message sent!") + +# Or use is_publish=false to get envelope and JSON without publishing +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=false) +# env: msg_envelope_v1 struct +# env_json_str: JSON string for publishing to NATS +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); + +// Send a text message +const data = [["message", "Hello World", "text"]]; +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat/room1", + data, + { broker_url: "nats://localhost:4222" } +); +// env: Object with all metadata and payloads +// env_json_str: JSON string for publishing +console.log("Message sent!"); + +// Or use is_publish=false +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat/room1", + data, + { broker_url: "nats://localhost:4222", is_publish: false } +); +``` + +#### Python + +```python +from natsbridge import smartsend + +# Send a text message +data = [("message", "Hello World", "text")] +env, env_json_str = await smartsend( + "/chat/room1", + data, + broker_url="nats://localhost:4222" +) +# env: Dict with all metadata and payloads +# env_json_str: JSON string for publishing +print("Message sent!") + +# Or use is_publish=False +env, env_json_str = await smartsend( + "/chat/room1", + data, + broker_url="nats://localhost:4222", + is_publish=False +) +# env: Dict with all metadata and payloads +# env_json_str: JSON string for publishing to NATS +``` + +#### MicroPython + +```python +from natsbridge_mpy import NATSBridge + +bridge = NATSBridge() + +# Send a text message (limited to small payloads) +data = [("message", "Hello World", "text")] +env, env_json_str = bridge.smartsend( + "/chat/room1", + data, + size_threshold=100000 # Lower threshold for MicroPython +) +print("Message sent!") +``` + +### Step 4: Receive Messages + +#### Julia + +```julia +using NATSBridge + +# Receive and process message +env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff) +# Returns: ::JSON.Object{String, Any} with "payloads" field containing Vector{Tuple{String, Any, String}} +# Access payloads: for (dataname, data, type) in env["payloads"] +for (dataname, data, type) in env["payloads"] + println("Received $dataname: $data") +end +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); + +// Receive and process message +const env = await NATSBridge.smartreceive(msg, { + fileserver_download_handler: NATSBridge.fetchWithBackoff +}); +// env.payloads = [[dataname, data, type], ...] +for (const [dataname, data, type] of env.payloads) { + console.log(`Received ${dataname}:`, data); +} +``` + +#### Python + +```python +from natsbridge import smartreceive, fetch_with_backoff + +# Receive and process message +env = await smartreceive( + msg, + fileserver_download_handler=fetch_with_backoff +) +# env["payloads"] = [(dataname, data, type), ...] +for dataname, data, type_ in env["payloads"]: + print(f"Received {dataname}: {data}") +``` + +--- + +## Basic Examples + +### Example 1: Sending a Dictionary + +#### Julia + +```julia +using NATSBridge + +config = Dict( + "wifi_ssid" => "MyNetwork", + "wifi_password" => "password123", + "update_interval" => 60 +) + +data = [("config", config, "dictionary")] +env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222") +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); + +const config = { + wifi_ssid: "MyNetwork", + wifi_password: "password123", + update_interval: 60 +}; + +const data = [["config", config, "dictionary"]]; +const [env, env_json_str] = await NATSBridge.smartsend( + "/device/config", + data, + { broker_url: "nats://localhost:4222" } +); +``` + +#### Python + +```python +from natsbridge import smartsend + +config = { + "wifi_ssid": "MyNetwork", + "wifi_password": "password123", + "update_interval": 60 +} + +data = [("config", config, "dictionary")] +env, env_json_str = await smartsend( + "/device/config", + data, + broker_url="nats://localhost:4222" +) +``` + +#### MicroPython + +```python +from natsbridge_mpy import NATSBridge + +bridge = NATSBridge() + +config = { + "wifi_ssid": "MyNetwork", + "wifi_password": "password123", + "update_interval": 60 +} + +data = [("config", config, "dictionary")] +env, env_json_str = bridge.smartsend( + "/device/config", + data, + size_threshold=100000 +) +``` + +### Example 2: Sending Binary Data (Image) + +#### Julia + +```julia +using NATSBridge + +# Read image file +image_data = read("image.png") + +data = [("user_image", image_data, "binary")] +env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222") +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); +const fs = require('fs'); + +// Read image file +const image_data = fs.readFileSync('image.png'); + +const data = [["user_image", image_data, "binary"]]; +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat/image", + data, + { broker_url: "nats://localhost:4222" } +); +``` + +#### Python + +```python +from natsbridge import smartsend + +# Read image file +with open("image.png", "rb") as f: + image_data = f.read() + +data = [("user_image", image_data, "binary")] +env, env_json_str = await smartsend( + "/chat/image", + data, + broker_url="nats://localhost:4222" +) +``` + +#### MicroPython + +```python +from natsbridge_mpy import NATSBridge + +bridge = NATSBridge() + +# Read image file +with open("image.png", "rb") as f: + image_data = f.read() + +data = [("user_image", image_data, "binary")] +env, env_json_str = bridge.smartsend( + "/chat/image", + data, + size_threshold=100000 +) +``` + +### Example 3: Request-Response Pattern + +#### Julia (Requester) + +```julia +using NATSBridge + +# Send command with reply-to +data = [("command", Dict("action" => "read_sensor"), "dictionary")] +env, env_json_str = smartsend( + "/device/command", + data, + broker_url="nats://localhost:4222", + reply_to="/device/response", + reply_to_msg_id="cmd-001" +) +``` + +#### JavaScript (Requester) + +```javascript +const NATSBridge = require('./src/natsbridge.js'); + +// Send command with reply-to +const data = [["command", { action: "read_sensor" }, "dictionary"]]; +const [env, env_json_str] = await NATSBridge.smartsend( + "/device/command", + data, + { + broker_url: "nats://localhost:4222", + reply_to: "/device/response", + reply_to_msg_id: "cmd-001" + } +); +``` + +#### Python (Requester) + +```python +from natsbridge import smartsend + +# Send command with reply-to +data = [("command", {"action": "read_sensor"}, "dictionary")] +env, env_json_str = await smartsend( + "/device/command", + data, + broker_url="nats://localhost:4222", + reply_to="/device/response", + reply_to_msg_id="cmd-001" +) +``` + +#### Julia (Responder) + +```julia +using NATSBridge, NATS + +const SUBJECT = "/device/command" +const NATS_URL = "nats://localhost:4222" + +function test_responder() + conn = NATS.connect(NATS_URL) + NATS.subscribe(conn, SUBJECT) do msg + env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) + + reply_to = env["reply_to"] + + for (dataname, data, type) in env["payloads"] + if dataname == "command" && data["action"] == "read_sensor" + response = Dict("sensor_id" => "sensor-001", "value" => 42.5) + if !isempty(reply_to) + smartsend(reply_to, [("data", response, "dictionary")]) + end + end + end + end + + sleep(120) + NATS.drain(conn) +end + +test_responder() +``` + +--- + +## Advanced Usage + +### Example 4: Large Payloads (File Server) + +For payloads larger than 1MB, NATSBridge automatically uses the file server: + +#### Julia + +```julia +using NATSBridge + +# Create large data (> 1MB) +large_data = rand(UInt8, 2_000_000) + +env, env_json_str = smartsend( + "/data/large", + [("large_file", large_data, "binary")], + broker_url="nats://localhost:4222", + fileserver_url="http://localhost:8080" +) + +println("File uploaded to: $(env.payloads[1].data)") +# Note: For link transport, data field contains the URL string +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); + +// Create large data (> 1MB) +const large_data = Buffer.alloc(2_000_000); +for (let i = 0; i < large_data.length; i++) { + large_data[i] = Math.floor(Math.random() * 256); +} + +const [env, env_json_str] = await NATSBridge.smartsend( + "/data/large", + [["large_file", large_data, "binary"]], + { + broker_url: "nats://localhost:4222", + fileserver_url: "http://localhost:8080" + } +); + +console.log("File uploaded to:", env.payloads[0].data); +// Note: For link transport, data field contains the URL string +``` + +#### Python + +```python +from natsbridge import smartsend + +# Create large data (> 1MB) +import os +large_data = os.urandom(2_000_000) + +env, env_json_str = await smartsend( + "/data/large", + [("large_file", large_data, "binary")], + broker_url="nats://localhost:4222", + fileserver_url="http://localhost:8080" +) + +print(f"File uploaded to: {env['payloads'][0]['data']}") +# Note: For link transport, data field contains the URL string +``` + +#### MicroPython + +MicroPython enforces a hard limit of 50KB per payload: + +```python +from natsbridge_mpy import NATSBridge + +bridge = NATSBridge() + +# MicroPython has a hard limit of 50KB per payload +# Use streaming or chunking for larger data +small_data = bytes(1000) # 1KB + +data = [("small_file", small_data, "binary")] +env, env_json_str = bridge.smartsend( + "/data/small", + data, + size_threshold=100000 # Enforced max: 50000 bytes +) +``` + +### Example 5: Mixed Content (Chat with Text + Image) + +NATSBridge supports sending multiple payloads with different types in a single message: + +#### Julia + +```julia +using NATSBridge + +image_data = read("avatar.png") + +data = [ + ("message_text", "Hello with image!", "text"), + ("user_avatar", image_data, "image") +] + +env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222") +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); +const fs = require('fs'); + +const image_data = fs.readFileSync('avatar.png'); + +const data = [ + ["message_text", "Hello with image!", "text"], + ["user_avatar", image_data, "image"] +]; + +const [env, env_json_str] = await NATSBridge.smartsend( + "/chat/mixed", + data, + { broker_url: "nats://localhost:4222" } +); +``` + +#### Python + +```python +from natsbridge import smartsend + +with open("avatar.png", "rb") as f: + image_data = f.read() + +data = [ + ("message_text", "Hello with image!", "text"), + ("user_avatar", image_data, "image") +] + +env, env_json_str = await smartsend( + "/chat/mixed", + data, + broker_url="nats://localhost:4222" +) +# env: Dict with all metadata and payloads +``` + +### Example 6: Table Data (Arrow IPC) + +For tabular data, NATSBridge uses Apache Arrow IPC format: + +#### Julia + +```julia +using NATSBridge +using DataFrames + +# Create DataFrame +df = DataFrame( + id = [1, 2, 3], + name = ["Alice", "Bob", "Charlie"], + score = [95, 88, 92] +) + +data = [("students", df, "arrowtable")] +env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222") +``` + +#### JavaScript + +```javascript +const NATSBridge = require('./src/natsbridge.js'); + +// Create table data (array of objects) +const table_data = [ + { id: 1, name: "Alice", score: 95 }, + { id: 2, name: "Bob", score: 88 }, + { id: 3, name: "Charlie", score: 92 } +]; + +const data = [["students", table_data, "arrowtable"]]; +const [env, env_json_str] = await NATSBridge.smartsend( + "/data/students", + data, + { broker_url: "nats://localhost:4222" } +); +``` + +#### Python + +```python +from natsbridge import smartsend +import pandas as pd + +# Create DataFrame +df = pd.DataFrame({ + 'id': [1, 2, 3], + 'name': ['Alice', 'Bob', 'Charlie'], + 'score': [95, 88, 92] +}) + +data = [("students", df, "table")] +env, env_json_str = await smartsend( + "/data/students", + data, + broker_url="nats://localhost:4222" +) +``` + +#### MicroPython + +MicroPython does not support table type due to memory constraints. Use dictionary or binary instead. + +--- + +## Next Steps + +1. **Explore the test directory** for more examples +2. **Check the documentation** for advanced configuration options +3. **Read the walkthrough** for building real-world applications + +--- + +## Troubleshooting + +### Connection Issues + +- Ensure NATS server is running: `docker ps | grep nats` +- Check firewall settings +- Verify NATS URL configuration + +### File Server Issues + +- Ensure file server is running and accessible +- Check upload permissions +- Verify file server URL configuration + +### Serialization Errors + +- Verify data type matches the specified type +- Check that binary data is in the correct format +- MicroPython: Ensure payload size < 50KB + +--- + +## License + +MIT diff --git a/docs/walkthrough.md b/docs/walkthrough.md new file mode 100644 index 0000000..39b7d1b --- /dev/null +++ b/docs/walkthrough.md @@ -0,0 +1,1378 @@ +# Cross-Platform NATSBridge Walkthrough + +A comprehensive guide to building real-world applications with NATSBridge across **Julia**, **JavaScript**, and **Python/MicroPython**. + +## Table of Contents + +1. [Introduction](#introduction) +2. [Architecture Overview](#architecture-overview) +3. [Building a Chat Application](#building-a-chat-application) +4. [Building a File Transfer System](#building-a-file-transfer-system) +5. [Building a Streaming Data Pipeline](#building-a-streaming-data-pipeline) +6. [Performance Optimization](#performance-optimization) +7. [Best Practices](#best-practices) + +--- + +## Introduction + +This walkthrough will guide you through building several real-world applications using NATSBridge. We'll cover: + +- Chat applications with rich media support +- File transfer systems with claim-check pattern +- Streaming data pipelines + +Each section builds on the previous one, gradually increasing in complexity. + +--- + +## Architecture Overview + +### Cross-Platform System Components + +```mermaid +flowchart TB + subgraph JuliaApp["Julia Application"] + JuliaAppCode[App Code] + JuliaBridge[NATSBridge.jl] + JuliaNATS[NATS.jl] + end + + subgraph JSApp["JavaScript Application"] + JSAppCode[App Code] + JSBridge[NATSBridge.js] + JSNATS[nats.js] + end + + subgraph PythonApp["Python Application"] + PythonAppCode[App Code] + PythonBridge[NATSBridge.py] + PythonNATS[nats-py] + end + + subgraph Infrastructure["Infrastructure"] + NATS[NATS Server
Message Broker] + FileServer[HTTP File Server
Upload/Download] + end + + JuliaAppCode --> JuliaBridge + JuliaBridge --> JuliaNATS + JSAppCode --> JSBridge + JSBridge --> JSNATS + PythonAppCode --> PythonBridge + PythonBridge --> PythonNATS + + JuliaNATS --> NATS + JSNATS --> NATS + PythonNATS --> NATS + + NATS --> JuliaNATS + NATS --> JSNATS + NATS --> PythonNATS + + JuliaBridge -.->|HTTP POST upload| FileServer + JSBridge -.->|HTTP POST upload| FileServer + PythonBridge -.->|HTTP POST upload| FileServer + + FileServer -.->|HTTP GET download| JuliaBridge + FileServer -.->|HTTP GET download| JSBridge + FileServer -.->|HTTP GET download| PythonBridge + + style JuliaApp fill:#c5e1a5 + style JSApp fill:#bbdefb + style PythonApp fill:#f8bbd0 + style NATS fill:#fff3e0 + style FileServer fill:#f3e5f5 +``` + +### Message Flow + +1. **Sender** creates a message envelope with payloads +2. **NATSBridge** serializes and encodes payloads +3. **Transport Decision**: Small payloads go directly to NATS, large payloads are uploaded to file server +4. **NATS** routes messages to subscribers +5. **Receiver** fetches payloads (from NATS or file server) +6. **NATSBridge** deserializes and decodes payloads + +--- + +## Building a Chat Application + +Let's build a full-featured chat application that supports text, images, and file attachments. + +### Step 1: Set Up the Project + +```bash +# Create project directory +mkdir -p chat-app/src +cd chat-app + +# Create configuration file +cat > config.json << 'EOF' +{ + "nats_url": "nats://localhost:4222", + "fileserver_url": "http://localhost:8080", + "size_threshold": 1048576 +} +EOF +``` + +### Step 2: Create the Chat Interface + +#### Julia + +```julia +# src/chat_ui.jl +using NATSBridge, NATS + +struct ChatUI + messages::Vector{Dict} + current_room::String +end + +function ChatUI() + ChatUI(Dict[], "") +end + +function send_message(ui::ChatUI, message_input::String, selected_file::Union{Nothing, String}) + data = [] + + # Add text message + if !isempty(message_input) + push!(data, ("text", message_input, "text")) + end + + # Add file if selected + if selected_file !== nothing + file_data = read(selected_file) + file_type = get_file_type(selected_file) + push!(data, ("attachment", file_data, file_type)) + end + + return data +end + +function get_file_type(filename::String)::String + if endswith(filename, ".png") || endswith(filename, ".jpg") + return "image" + elseif endswith(filename, ".mp3") || endswith(filename, ".wav") + return "audio" + elseif endswith(filename, ".mp4") || endswith(filename, ".avi") + return "video" + else + return "binary" + end +end + +function add_message(ui::ChatUI, user::String, text::String, attachment::Union{Nothing, Dict}) + push!(ui.messages, Dict( + "user" => user, + "text" => text, + "attachment" => attachment + )) +end +``` + +#### JavaScript + +```javascript +// src/chat_ui.js +const NATSBridge = require('./src/natsbridge.js'); + +class ChatUI { + constructor() { + this.messages = []; + this.currentRoom = ""; + } + + sendMessage(messageInput, selectedFile = null) { + const data = []; + + // Add text message + if (messageInput.length > 0) { + data.push(["text", messageInput, "text"]); + } + + // Add file if selected + if (selectedFile !== null) { + const fileData = fs.readFileSync(selectedFile); + const fileType = this.getFileType(selectedFile); + data.push(["attachment", fileData, fileType]); + } + + return data; + } + + getFileType(filename) { + if (filename.endsWith('.png') || filename.endsWith('.jpg')) { + return 'image'; + } else if (filename.endsWith('.mp3') || filename.endsWith('.wav')) { + return 'audio'; + } else if (filename.endsWith('.mp4') || filename.endsWith('.avi')) { + return 'video'; + } else { + return 'binary'; + } + } + + addMessage(user, text, attachment = null) { + this.messages.push({ + user, + text, + attachment + }); + } +} + +module.exports = ChatUI; +``` + +#### Python + +```python +# src/chat_ui.py +from typing import List, Dict, Optional, Union + +class ChatUI: + def __init__(self): + self.messages: List[Dict] = [] + self.current_room: str = "" + + def send_message(self, message_input: str, selected_file: Optional[str] = None) -> List[tuple]: + data = [] + + # Add text message + if message_input: + data.append(("text", message_input, "text")) + + # Add file if selected + if selected_file: + with open(selected_file, "rb") as f: + file_data = f.read() + file_type = self.get_file_type(selected_file) + data.append(("attachment", file_data, file_type)) + + return data + + def get_file_type(self, filename: str) -> str: + if filename.endswith(('.png', '.jpg')): + return "image" + elif filename.endswith(('.mp3', '.wav')): + return "audio" + elif filename.endswith(('.mp4', '.avi')): + return "video" + else: + return "binary" + + def add_message(self, user: str, text: str, attachment: Optional[Dict] = None): + self.messages.append({ + "user": user, + "text": text, + "attachment": attachment + }) +``` + +### Step 3: Create the Message Handler + +#### Julia + +```julia +# src/chat_handler.jl +using NATSBridge, NATS + +struct ChatHandler + nats::NATS.Connection + ui::ChatUI +end + +function ChatHandler(nats_connection::NATS.Connection) + ChatHandler(nats_connection, ChatUI()) +end + +function start(handler::ChatHandler) + # Subscribe to chat rooms + rooms = ["general", "tech", "random"] + + for room in rooms + NATS.subscribe(handler.nats, "/chat/$room") do msg + handle_message(handler, msg) + end + end + + println("Chat handler started") +end + +function handle_message(handler::ChatHandler, msg::NATS.Msg) + env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) + + # Extract sender info from envelope + sender = get(env, "sender_name", "Anonymous") + + # Process each payload + for (dataname, data, type) in env["payloads"] + if type == "text" + add_message(handler.ui, sender, data, nothing) + elseif type == "image" + # Convert to data URL for display + base64_data = base64encode(data) + attachment = Dict( + "type" => "image", + "data" => "data:image/png;base64,$base64_data" + ) + add_message(handler.ui, sender, "", attachment) + else + # For other types, use file server URL + attachment = Dict("type" => type, "data" => data) + add_message(handler.ui, sender, "", attachment) + end + end +end +``` + +#### JavaScript + +```javascript +// src/chat_handler.js +const NATSBridge = require('./src/natsbridge.js'); +const nats = require('nats'); + +class ChatHandler { + constructor(natsConnection) { + this.nats = natsConnection; + this.ui = new (require('./chat_ui.js'))(); + } + + async start() { + // Subscribe to chat rooms + const rooms = ['general', 'tech', 'random']; + + for (const room of rooms) { + this.nats.subscribe(`/chat/${room}`, async (msg) => { + await this.handleMessage(msg); + }); + } + + console.log('Chat handler started'); + } + + async handleMessage(msg) { + const env = await NATSBridge.smartreceive(msg, { + fileserver_download_handler: NATSBridge.fetchWithBackoff + }); + + // Extract sender info from envelope + const sender = env.sender_name || 'Anonymous'; + + // Process each payload + for (const [dataname, data, type] of env.payloads) { + if (type === 'text') { + this.ui.addMessage(sender, data, null); + } else if (type === 'image') { + // Convert to data URL for display + const base64Data = Buffer.from(data).toString('base64'); + const attachment = { + type: 'image', + data: `data:image/png;base64,${base64Data}` + }; + this.ui.addMessage(sender, '', attachment); + } else { + // For other types, use file server URL + const attachment = { type, data }; + this.ui.addMessage(sender, '', attachment); + } + } + } +} + +module.exports = ChatHandler; +``` + +#### Python + +```python +# src/chat_handler.py +import asyncio +from typing import Optional +from natsbridge import smartreceive, fetch_with_backoff + +class ChatHandler: + def __init__(self, nats_connection): + self.nats = nats_connection + self.ui = ChatUI() + + async def start(self): + # Subscribe to chat rooms + rooms = ['general', 'tech', 'random'] + + for room in rooms: + await self.nats.subscribe( + f'/chat/{room}', + callback=self.handle_message + ) + + print('Chat handler started') + + async def handle_message(self, msg): + env = await smartreceive( + msg, + fileserver_download_handler=fetch_with_backoff + ) + + # Extract sender info from envelope + sender = env.get('sender_name', 'Anonymous') + + # Process each payload + for dataname, data, type_ in env['payloads']: + if type_ == 'text': + self.ui.add_message(sender, data, None) + elif type_ == 'image': + # Convert to data URL for display + import base64 + base64_data = base64.b64encode(data).decode('utf-8') + attachment = { + 'type': 'image', + 'data': f'data:image/png;base64,{base64_data}' + } + self.ui.add_message(sender, '', attachment) + else: + # For other types, use file server URL or data + attachment = {'type': type_, 'data': data} + self.ui.add_message(sender, '', attachment) +``` + +### Step 4: Run the Application + +```bash +# Start NATS +docker run -p 4222:4222 nats:latest + +# Start file server +mkdir -p /tmp/fileserver +python3 -m http.server 8080 --directory /tmp/fileserver + +# Run chat app +# Julia +julia src/chat_ui.jl +julia src/chat_handler.jl + +# JavaScript +node src/chat_ui.js +node src/chat_handler.js + +# Python +python3 src/chat_ui.py +python3 src/chat_handler.py +``` + +--- + +## Building a File Transfer System + +Let's build a file transfer system that handles large files efficiently. + +### Step 1: File Upload Service + +#### Julia + +```julia +# src/file_upload_service.jl +using NATSBridge, HTTP + +struct FileUploadService + broker_url::String + fileserver_url::String +end + +function FileUploadService(broker_url::String, fileserver_url::String) + FileUploadService(broker_url, fileserver_url) +end + +function upload_file(service::FileUploadService, file_path::String, recipient::String)::Dict + file_data = read(file_path) + file_name = basename(file_path) + + data = [("file", file_data, "binary")] + + env, env_json_str = smartsend( + "/files/$recipient", + data, + broker_url=service.broker_url, + fileserver_url=service.fileserver_url + ) + + return env +end + +function upload_large_file(service::FileUploadService, file_path::String, recipient::String)::Dict + file_size = stat(file_path).size + + if file_size > 100 * 1024 * 1024 # > 100MB + println("File too large for direct upload, using streaming...") + return stream_upload(service, file_path, recipient) + end + + return upload_file(service, file_path, recipient) +end + +function stream_upload(service::FileUploadService, file_path::String, recipient::String)::Dict + # Implement streaming upload to file server + # This would require a more sophisticated file server + # For now, we'll use the standard upload + return upload_file(service, file_path, recipient) +end +``` + +#### JavaScript + +```javascript +// src/file_upload_service.js +const NATSBridge = require('./src/natsbridge.js'); +const fs = require('fs'); + +class FileUploadService { + constructor(brokerUrl, fileserverUrl) { + this.broker_url = brokerUrl; + this.fileserver_url = fileserverUrl; + } + + async uploadFile(filePath, recipient) { + const fileData = fs.readFileSync(filePath); + const fileName = require('path').basename(filePath); + + const data = [["file", fileData, "binary"]]; + + const [env, env_json_str] = await NATSBridge.smartsend( + `/files/${recipient}`, + data, + { + broker_url: this.broker_url, + fileserver_url: this.fileserver_url + } + ); + + return env; + } + + async uploadLargeFile(filePath, recipient) { + const stats = fs.statSync(filePath); + const fileSize = stats.size; + + if (fileSize > 100 * 1024 * 1024) { // > 100MB + console.log('File too large for direct upload, using streaming...'); + return this.streamUpload(filePath, recipient); + } + + return this.uploadFile(filePath, recipient); + } + + async streamUpload(filePath, recipient) { + // Implement streaming upload to file server + // This would require a more sophisticated file server + // For now, we'll use the standard upload + return this.uploadFile(filePath, recipient); + } +} + +module.exports = FileUploadService; +``` + +#### Python + +```python +# src/file_upload_service.py +from natsbridge import smartsend +import os + +class FileUploadService: + def __init__(self, broker_url: str, fileserver_url: str): + self.broker_url = broker_url + self.fileserver_url = fileserver_url + + async def upload_file(self, file_path: str, recipient: str) -> tuple: + with open(file_path, "rb") as f: + file_data = f.read() + file_name = os.path.basename(file_path) + + data = [("file", file_data, "binary")] + + env, env_json_str = await smartsend( + f"/files/{recipient}", + data, + broker_url=self.broker_url, + fileserver_url=self.fileserver_url + ) + + return env, env_json_str + + async def upload_large_file(self, file_path: str, recipient: str) -> tuple: + file_size = os.path.getsize(file_path) + + if file_size > 100 * 1024 * 1024: # > 100MB + print("File too large for direct upload, using streaming...") + return await self.stream_upload(file_path, recipient) + + return await self.upload_file(file_path, recipient) + + async def stream_upload(self, file_path: str, recipient: str) -> tuple: + # Implement streaming upload to file server + # This would require a more sophisticated file server + # For now, we'll use the standard upload + return await self.upload_file(file_path, recipient) +``` + +### Step 2: File Download Service + +#### Julia + +```julia +# src/file_download_service.jl +using NATSBridge + +struct FileDownloadService + nats_url::String +end + +function FileDownloadService(nats_url::String) + FileDownloadService(nats_url) +end + +function download_file(service::FileDownloadService, msg::NATS.Msg, sender::String, download_id::String) + env = smartreceive(msg, fileserver_download_handler=fetch_from_url) + + # Process each payload + for (dataname, data, type) in env["payloads"] + if type == "binary" + file_path = "/downloads/$dataname" + write(file_path, data) + println("File saved to $file_path") + end + end +end + +function fetch_from_url(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} + # Fetch data from URL with exponential backoff + # Return downloaded data as Vector{UInt8} +end +``` + +#### JavaScript + +```javascript +// src/file_download_service.js +const NATSBridge = require('./src/natsbridge.js'); +const fs = require('fs'); + +class FileDownloadService { + constructor(natsUrl) { + this.nats_url = natsUrl; + } + + async downloadFile(msg, sender, downloadId) { + const env = await NATSBridge.smartreceive(msg, { + fileserver_download_handler: NATSBridge.fetchWithBackoff + }); + + // Process each payload + for (const [dataname, data, type] of env.payloads) { + if (type === 'binary') { + const filePath = `/downloads/${dataname}`; + fs.writeFileSync(filePath, data); + console.log(`File saved to ${filePath}`); + } + } + } +} + +module.exports = FileDownloadService; +``` + +#### Python + +```python +# src/file_download_service.py +from natsbridge import smartreceive, fetch_with_backoff +import os + +class FileDownloadService: + def __init__(self, nats_url: str): + self.nats_url = nats_url + + async def download_file(self, msg, sender: str, download_id: str): + env = await smartreceive( + msg, + fileserver_download_handler=fetch_with_backoff + ) + + # Process each payload + for dataname, data, type_ in env['payloads']: + if type_ == 'binary': + file_path = f'/downloads/{dataname}' + os.makedirs('/downloads', exist_ok=True) + with open(file_path, 'wb') as f: + f.write(data) + print(f"File saved to {file_path}") +``` + +### Step 3: File Transfer CLI + +#### Julia + +```julia +# src/cli.jl +using NATSBridge + +function main() + println("File Transfer System") + println("====================") + println("1. Upload file") + println("2. Download file") + println("3. List pending downloads") + + print("Enter choice: ") + choice = readline() + + if choice == "1" + upload_file_cli() + elseif choice == "2" + download_file_cli() + end +end + +function upload_file_cli() + print("Enter file path: ") + file_path = readline() + + print("Enter recipient: ") + recipient = readline() + + file_service = FileUploadService("nats://localhost:4222", "http://localhost:8080") + + try + env = upload_file(file_service, file_path, recipient) + println("Upload successful!") + println("File ID: $(env["payloads"][1][1])") + catch error + println("Upload failed: $(error)") + end +end + +function download_file_cli() + print("Enter sender: ") + sender = readline() + + file_service = FileDownloadService("nats://localhost:4222") + + try + download_file(file_service, sender) + println("Download complete!") + catch error + println("Download failed: $(error)") + end +end + +main() +``` + +--- + +## Building a Streaming Data Pipeline + +Let's build a data pipeline that processes streaming data from sensors. + +### Step 1: Sensor Data Model + +#### Julia + +```julia +# src/sensor_data.jl +using Dates, DataFrames + +struct SensorReading + sensor_id::String + timestamp::String + value::Float64 + unit::String + metadata::Dict{String, Any} +end + +function SensorReading(sensor_id::String, value::Float64, unit::String, metadata::Dict{String, Any}=Dict()) + SensorReading( + sensor_id, + ISODateTime(now(), Dates.Second) |> string, + value, + unit, + metadata + ) +end + +struct SensorBatch + readings::Vector{SensorReading} +end + +function SensorBatch() + SensorBatch(SensorReading[]) +end + +function add_reading(batch::SensorBatch, reading::SensorReading) + push!(batch.readings, reading) +end + +function to_dataframe(batch::SensorBatch)::DataFrame + data = Dict{String, Any}() + data["sensor_id"] = [r.sensor_id for r in batch.readings] + data["timestamp"] = [r.timestamp for r in batch.readings] + data["value"] = [r.value for r in batch.readings] + data["unit"] = [r.unit for r in batch.readings] + + return DataFrame(data) +end +``` + +#### JavaScript + +```javascript +// src/sensor_data.js +const NATSBridge = require('./src/natsbridge.js'); + +class SensorReading { + constructor(sensorId, value, unit, metadata = {}) { + this.sensor_id = sensorId; + this.timestamp = new Date().toISOString(); + this.value = value; + this.unit = unit; + this.metadata = metadata; + } +} + +class SensorBatch { + constructor() { + this.readings = []; + } + + addReading(reading) { + this.readings.push(reading); + } + + toDataFrame() { + return { + sensor_id: this.readings.map(r => r.sensor_id), + timestamp: this.readings.map(r => r.timestamp), + value: this.readings.map(r => r.value), + unit: this.readings.map(r => r.unit) + }; + } +} + +module.exports = { SensorReading, SensorBatch }; +``` + +#### Python + +```python +# src/sensor_data.py +from datetime import datetime +from dataclasses import dataclass, field +from typing import List, Dict, Any + +@dataclass +class SensorReading: + sensor_id: str + timestamp: str + value: float + unit: str + metadata: Dict[str, Any] = field(default_factory=dict) + + @classmethod + def create(cls, sensor_id: str, value: float, unit: str, metadata: Dict[str, Any] = None): + return cls( + sensor_id=sensor_id, + timestamp=datetime.utcnow().isoformat(), + value=value, + unit=unit, + metadata=metadata or {} + ) + +class SensorBatch: + def __init__(self): + self.readings: List[SensorReading] = [] + + def add_reading(self, reading: SensorReading): + self.readings.append(reading) + + def to_dataframe(self): + import pandas as pd + return pd.DataFrame({ + 'sensor_id': [r.sensor_id for r in self.readings], + 'timestamp': [r.timestamp for r in self.readings], + 'value': [r.value for r in self.readings], + 'unit': [r.unit for r in self.readings] + }) +``` + +### Step 2: Sensor Sender + +#### Julia + +```julia +# src/sensor_sender.jl +using NATSBridge, Dates, Random + +struct SensorSender + broker_url::String + fileserver_url::String +end + +function SensorSender(broker_url::String, fileserver_url::String) + SensorSender(broker_url, fileserver_url) +end + +function send_reading(sender::SensorSender, sensor_id::String, value::Float64, unit::String) + reading = SensorReading(sensor_id, value, unit) + + data = [("reading", reading.metadata, "dictionary")] + + # Default: is_publish=True (automatically publishes to NATS) + smartsend( + "/sensors/$sensor_id", + data, + broker_url=sender.broker_url, + fileserver_url=sender.fileserver_url + ) +end + +function send_batch(sender::SensorSender, readings::Vector{SensorReading}) + batch = SensorBatch() + for reading in readings + add_reading(batch, reading) + end + + df = to_dataframe(batch) + + # Convert to Arrow IPC format + import Arrow + table = Arrow.Table(df) + + # Serialize to Arrow IPC + import IOBuffer + buf = IOBuffer() + Arrow.write(buf, table) + + arrow_data = take!(buf) + + # Send based on size (auto-selected by smartsend) + data = [("batch", arrow_data, "arrowtable")] + smartsend( + "/sensors/batch", + data, + broker_url=sender.broker_url, + fileserver_url=sender.fileserver_url + ) +end +``` + +#### JavaScript + +```javascript +// src/sensor_sender.js +const NATSBridge = require('./src/natsbridge.js'); +const { SensorReading, SensorBatch } = require('./sensor_data.js'); + +class SensorSender { + constructor(brokerUrl, fileserverUrl) { + this.broker_url = brokerUrl; + this.fileserver_url = fileserverUrl; + } + + async sendReading(sensorId, value, unit) { + const reading = new SensorReading(sensorId, value, unit); + + const data = [["reading", reading.metadata, "dictionary"]]; + + await NATSBridge.smartsend( + `/sensors/${sensorId}`, + data, + { + broker_url: this.broker_url, + fileserver_url: this.fileserver_url + } + ); + } + + async sendBatch(readings) { + const batch = new SensorBatch(); + for (const reading of readings) { + batch.addReading(reading); + } + + const df = batch.toDataFrame(); + + // Convert to Arrow IPC + const arrow = require('apache-arrow'); + const schema = new arrow.Schema([ + new arrow.Field('sensor_id', arrow.string()), + new arrow.Field('timestamp', arrow.string()), + new arrow.Field('value', arrow.float64()), + new arrow.Field('unit', arrow.string()) + ]); + + const arrays = { + sensor_id: new arrow.StringArray(df.sensor_id.map(s => String(s))), + timestamp: new arrow.StringArray(df.timestamp), + value: new arrow.Float64Array(df.value), + unit: new arrow.StringArray(df.unit) + }; + + const recordBatch = arrow.RecordBatch.fromArrays(schema, arrays, df.value.length); + const buffer = arrow.tableFromBatches([recordBatch]).toBuffer(); + const arrow_data = new Uint8Array(buffer); + + // Send based on size (auto-selected by smartsend) + const data = [["batch", arrow_data, "arrowtable"]]; + await NATSBridge.smartsend( + "/sensors/batch", + data, + { + broker_url: this.broker_url, + fileserver_url: this.fileserver_url + } + ); + } +} + +module.exports = SensorSender; +``` + +#### Python + +```python +# src/sensor_sender.py +from natsbridge import smartsend +from sensor_data import SensorReading, SensorBatch + +class SensorSender: + def __init__(self, broker_url: str, fileserver_url: str): + self.broker_url = broker_url + self.fileserver_url = fileserver_url + + async def send_reading(self, sensor_id: str, value: float, unit: str): + reading = SensorReading.create(sensor_id, value, unit) + + data = [("reading", reading.metadata, "dictionary")] + + await smartsend( + f"/sensors/{sensor_id}", + data, + broker_url=self.broker_url, + fileserver_url=self.fileserver_url + ) + + async def send_batch(self, readings): + batch = SensorBatch() + for reading in readings: + batch.add_reading(reading) + + df = batch.to_dataframe() + + # Convert to Arrow IPC + import pyarrow as arrow + import pyarrow.ipc as ipc + import io + + table = arrow.Table.from_pandas(df) + buf = io.BytesIO() + sink = ipc.new_file(buf, table.schema) + ipc.write_table(table, sink) + sink.close() + arrow_data = buf.getvalue() + + # Send based on size (auto-selected by smartsend) + data = [("batch", arrow_data, "arrowtable")] + await smartsend( + "/sensors/batch", + data, + broker_url=self.broker_url, + fileserver_url=self.fileserver_url + ) +``` + +--- + +## Performance Optimization + +### 1. Batch Processing + +#### Julia + +```julia +# Batch multiple readings into a single message +function send_batch_readings(sender::SensorSender, readings::Vector{Tuple{String, Float64, String}}) + batch = SensorBatch() + + for (sensor_id, value, unit) in readings + reading = SensorReading(sensor_id, value, unit) + add_reading(batch, reading) + end + + df = to_dataframe(batch) + + # Convert to Arrow IPC + import Arrow + table = Arrow.Table(df) + + # Serialize to Arrow IPC + import IOBuffer + buf = IOBuffer() + Arrow.write(buf, table) + + arrow_data = take!(buf) + + # Send as single message + smartsend( + "/sensors/batch", + [("batch", arrow_data, "arrowtable")], + broker_url=sender.broker_url + ) +end +``` + +#### JavaScript + +```javascript +// Batch multiple readings into a single message +async function sendBatchReadings(sender, readings) { + const batch = new SensorBatch(); + + for (const [sensorId, value, unit] of readings) { + const reading = new SensorReading(sensorId, value, unit); + batch.addReading(reading); + } + + const df = batch.toDataFrame(); + + // Convert to Arrow IPC + const arrow = require('apache-arrow'); + const schema = new arrow.Schema([ + new arrow.Field('sensor_id', arrow.string()), + new arrow.Field('timestamp', arrow.string()), + new arrow.Field('value', arrow.float64()), + new arrow.Field('unit', arrow.string()) + ]); + + const arrays = { + sensor_id: new arrow.StringArray(df.sensor_id), + timestamp: new arrow.StringArray(df.timestamp), + value: new arrow.Float64Array(df.value), + unit: new arrow.StringArray(df.unit) + }; + + const recordBatch = arrow.RecordBatch.fromArrays(schema, arrays, df.value.length); + const buffer = arrow.tableFromBatches([recordBatch]).toBuffer(); + const arrow_data = new Uint8Array(buffer); + + // Send as single message + const data = [["batch", arrow_data, "arrowtable"]]; + await NATSBridge.smartsend( + "/sensors/batch", + data, + { broker_url: sender.broker_url } + ); +} +``` + +### 2. Connection Reuse + +#### Julia + +```julia +# Reuse NATS connections +function create_connection_pool() + connections = Dict{String, NATS.Connection}() + + function get_connection(nats_url::String)::NATS.Connection + if !haskey(connections, nats_url) + connections[nats_url] = NATS.connect(nats_url) + end + return connections[nats_url] + end + + function close_all() + for conn in values(connections) + NATS.drain(conn) + end + empty!(connections) + end + + return (get_connection=get_connection, close_all=close_all) +end +``` + +#### Python + +```python +# Reuse NATS connections +import asyncio +import nats + +class ConnectionPool: + def __init__(self): + self.connections = {} + + async def get_connection(self, nats_url: str): + if nats_url not in self.connections: + self.connections[nats_url] = await nats.connect(nats_url) + return self.connections[nats_url] + + async def close_all(self): + for conn in self.connections.values(): + await conn.drain() + self.connections.clear() +``` + +### 3. Caching + +#### Julia + +```julia +# Cache file server responses +using Base.Threads + +const file_cache = Dict{String, Vector{UInt8}}() + +function fetch_with_caching(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} + if haskey(file_cache, url) + return file_cache[url] + end + + # Fetch from file server + data = _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) + + # Cache the result + file_cache[url] = data + + return data +end +``` + +#### Python + +```python +# Cache file server responses +import asyncio +import threading +from natsbridge import fetch_with_backoff + +file_cache = {} +cache_lock = threading.Lock() + +async def fetch_with_caching(url, max_retries, base_delay, max_delay, correlation_id): + with cache_lock: + if url in file_cache: + return file_cache[url] + + # Fetch from file server + data = await fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) + + # Cache the result + with cache_lock: + file_cache[url] = data + + return data +``` + +--- + +## Best Practices + +### 1. Error Handling + +#### Julia + +```julia +function safe_smartsend(subject::String, data::Vector{Tuple}, kwargs...) + try + return smartsend(subject, data; kwargs...) + catch error + println("Failed to send message: $(error)") + return nothing + end +end +``` + +#### JavaScript + +```javascript +async function safeSmartSend(subject, data, options = {}) { + try { + return await NATSBridge.smartsend(subject, data, options); + } catch (error) { + console.error(`Failed to send message: ${error}`); + return null; + } +} +``` + +#### Python + +```python +from typing import List, Tuple, Optional, Union + +async def safe_smartsend( + subject: str, + data: List[Tuple[str, Any, str]], + **kwargs +) -> Optional[Tuple[dict, str]]: + try: + return await smartsend(subject, data, **kwargs) + except Exception as error: + print(f"Failed to send message: {error}") + return None +``` + +### 2. Logging + +#### Julia + +```julia +using Logging + +function log_send(subject::String, data::Vector{Tuple}, correlation_id::String) + @info "Sending to $subject: $(length(data)) payloads, correlation_id=$correlation_id" +end + +function log_receive(correlation_id::String, num_payloads::Int) + @info "Received message: $num_payloads payloads, correlation_id=$correlation_id" +end +``` + +#### Python + +```python +import logging +from typing import List, Tuple, Any + +logger = logging.getLogger(__name__) + +def log_send(subject: str, data: List[Tuple[str, Any, str]], correlation_id: str): + logger.info(f"Sending to {subject}: {len(data)} payloads, correlation_id={correlation_id}") + +def log_receive(correlation_id: str, num_payloads: int): + logger.info(f"Received message: {num_payloads} payloads, correlation_id={correlation_id}") +``` + +--- + +## Conclusion + +This walkthrough covered: + +- Building a chat application with rich media support +- Building a file transfer system with claim-check pattern +- Building a streaming data pipeline for sensor data + +For more information, check the [API documentation](../src/README.md) and [test examples](../test/). + +--- + +## License + +MIT diff --git a/etc.jl b/etc.jl deleted file mode 100644 index 878e279..0000000 --- a/etc.jl +++ /dev/null @@ -1,9 +0,0 @@ -Task: Update README.md to reflect recent changes in NATSbridge package. - -Context: the package has been updated with the NATS_connection keyword and the publish_message function. - -Requirements: - -Source of Truth: Treat the updated NATSbridge code as the definitive source. Update README.md to align exactly with these changes. -API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. -Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. \ No newline at end of file diff --git a/etc.txt b/etc.txt new file mode 100644 index 0000000..074b9bc --- /dev/null +++ b/etc.txt @@ -0,0 +1,310 @@ +#!/usr/bin/env julia +# Test script for mixed-content message testing +# Tests receiving a mix of text, json, table, image, audio, video, and binary data +# from Julia serviceA to Julia serviceB using NATSBridge.jl smartreceive +# +# This test demonstrates that any combination and any number of mixed content +# can be sent and received correctly. + +using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64 + +# Include the bridge module +include("./src/NATSBridge.jl") +using .NATSBridge + +# Configuration +const SUBJECT = "/test/mix" +const NATS_URL = "nats.yiem.cc" +const FILESERVER_URL = "http://192.168.88.104:8080" + + +# ------------------------------------------------------------------------------------------------ # +# test mixed content transfer # +# ------------------------------------------------------------------------------------------------ # + + +# Helper: Log with correlation ID +function log_trace(message) + timestamp = Dates.now() + println("[$timestamp] $message") +end + + +# Receiver: Listen for messages and verify mixed content handling +function test_mix_receive() + conn = NATS.connect(NATS_URL) + incoming_msg = nothing + NATS.subscribe(conn, SUBJECT) do msg + log_trace("Received message on $(msg.subject)") + incoming_msg = msg + + # # Use NATSBridge.smartreceive to handle the data + # # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) + # result = NATSBridge.smartreceive( + # msg; + # max_retries = 5, + # base_delay = 100, + # max_delay = 5000 + # ) + + # log_trace("Received $(length(result["payloads"])) payloads") + + + # # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples + # for (dataname, data, data_type) in result["payloads"] + # log_trace("\n=== Payload: $dataname (type: $data_type) ===") + + # # Handle different data types + # if data_type == "text" + # # Text data - should be a String + # if isa(data, String) + # log_trace(" Type: String") + # log_trace(" Length: $(length(data)) characters") + + # # Display first 200 characters + # if length(data) > 200 + # log_trace(" First 200 chars: $(data[1:200])...") + # else + # log_trace(" Content: $data") + # end + + # # Save to file + # output_path = "./received_$dataname.txt" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected String, got $(typeof(data))") + # end + + # elseif data_type == "dictionary" + # # Dictionary data - should be JSON object + # if isa(data, JSON.Object{String, Any}) + # log_trace(" Type: Dict") + # log_trace(" Keys: $(keys(data))") + + # # Display nested content + # for (key, value) in data + # log_trace(" $key => $value") + # end + + # # Save to JSON file + # output_path = "./received_$dataname.json" + # json_str = JSON.json(data, 2) + # write(output_path, json_str) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Dict, got $(typeof(data))") + # end + + # elseif data_type == "table" + # # Table data - should be a DataFrame + # tabledata = deepcopy(data) + # println("found table data") + # break + # # return data + # # if isa(data, DataFrame) + # # log_trace(" Type: DataFrame") + # # log_trace(" Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns") + # # log_trace(" Columns: $(names(data))") + + # # # Display first few rows + # # log_trace(" First 5 rows:") + # # display(data[1:min(5, size(data, 1)), :]) + + # # # Save to Arrow file + # # output_path = "./received_$dataname.arrow" + # # io = IOBuffer() + # # Arrow.write(io, data) + # # write(output_path, take!(io)) + # # log_trace(" Saved to: $output_path") + # # else + # # log_trace(" ERROR: Expected DataFrame, got $(typeof(data))") + # # end + + # elseif data_type == "image" + # # Image data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # elseif data_type == "audio" + # # Audio data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # elseif data_type == "video" + # # Video data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # elseif data_type == "binary" + # # Binary data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # else + # log_trace(" ERROR: Unknown data type '$data_type'") + # end + # end + + # Summary + # println("\n=== Verification Summary ===") + # text_count = count(x -> x[3] == "text", result["payloads"]) + # dict_count = count(x -> x[3] == "dictionary", result["payloads"]) + # table_count = count(x -> x[3] == "table", result["payloads"]) + # image_count = count(x -> x[3] == "image", result["payloads"]) + # audio_count = count(x -> x[3] == "audio", result["payloads"]) + # video_count = count(x -> x[3] == "video", result["payloads"]) + # binary_count = count(x -> x[3] == "binary", result["payloads"]) + + # log_trace("Text payloads: $text_count") + # log_trace("Dictionary payloads: $dict_count") + # log_trace("Table payloads: $table_count") + # log_trace("Image payloads: $image_count") + # log_trace("Audio payloads: $audio_count") + # log_trace("Video payloads: $video_count") + # log_trace("Binary payloads: $binary_count") + + # # Print transport type info for each payload if available + # println("\n=== Payload Details ===") + # for (dataname, data, data_type) in result["payloads"] + # if data_type in ["image", "audio", "video", "binary"] + # log_trace("$dataname: $(length(data)) bytes (binary)") + # elseif data_type == "table" + # data = DataFrame(data) + # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)") + # elseif data_type == "dictionary" + # log_trace("$dataname: $(length(JSON.json(data))) bytes (Dict)") + # elseif data_type == "text" + # log_trace("$dataname: $(length(data)) characters (String)") + # end + # end + end + + # Keep listening for 2 minutes + sleep(20) + NATS.drain(conn) + return incoming_msg +end + + +# Run the test +println("Starting mixed-content transport test...") +println("Note: This receiver will wait for messages from the sender.") +println("Run test_julia_to_julia_mix_sender.jl first to send test data.") + +# Run receiver +println("\ntesting smartreceive for mixed content") +incoming_msg = test_mix_receive() + +println("\nTest completed.") + + + + + + + + + + + + + + +Check architecture.md. For sending table I want to add JSON in addition to Apache Arrow. +Currently I use "table" datatype when sending table data using Arrow. Now table that I want to send using JSON +I will use "jsontable" as datatype while sending table using Arrow I will use "arrowtable" as datatype. +This will select how smartsend and smartreceive serialize/deserialize the table. + +Can you help me do this? Save the updated architecture.md into updated_architecture.md file. I will deal with source code later. + + + + +Now update implementation.md and save into updated_implementation.md +Keep in mind that Julia DataFrame and Python Pandas rely on columnar-oriented dictionary to create as the following example: +julia> dict = Dict("customer age" => [15, 20, 25], + "first name" => ["Rohit", "Rahul", "Akshat"]) +julia> DataFrame(dict) + +python> data = { + "Name": ["Alice", "Bob", "Charlie"], + "Age": [25, 30, 35], + "Score": [88.5, 92.0, 79.5] +} + +python> df = pd.DataFrame(data) + + +But JS use Array of Objects while MicroPython use list of lists. Both are row-oriented structure. +So use row-oriented JSON to send across these languages. For Julia and Python, only convert +row-oriented JSON to columnar-oriented dictionary for "going-into" and vise versa for "coming-out" +a dataframe while JS and MicroPython won't require such process. +You may add these info into architecture.md if you see fit. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/tutorial.md b/examples/tutorial.md deleted file mode 100644 index 22ebbdf..0000000 --- a/examples/tutorial.md +++ /dev/null @@ -1,305 +0,0 @@ -# NATSBridge Tutorial - -A step-by-step guide to get started with NATSBridge - a high-performance, bi-directional data bridge for **Julia**. - -## Table of Contents - -1. [Overview](#overview) -2. [Prerequisites](#prerequisites) -3. [Installation](#installation) -4. [Quick Start](#quick-start) -5. [Basic Examples](#basic-examples) -6. [Advanced Usage](#advanced-usage) - ---- - -## Overview - -NATSBridge enables seamless communication for Julia applications through NATS, with automatic transport selection based on payload size: - -- **Direct Transport**: Payloads < 1MB are sent directly via NATS (Base64 encoded) -- **Link Transport**: Payloads >= 1MB are uploaded to an HTTP file server and referenced via URL - -### Supported Payload Types - -| Type | Description | -|------|-------------| -| `text` | Plain text strings | -| `dictionary` | JSON-serializable dictionaries | -| `table` | Tabular data (Arrow IPC format) | -| `image` | Image data (PNG, JPG bytes) | -| `audio` | Audio data (WAV, MP3 bytes) | -| `video` | Video data (MP4, AVI bytes) | -| `binary` | Generic binary data | - ---- - -## Prerequisites - -Before you begin, ensure you have: - -1. **NATS Server** running (or accessible) -2. **HTTP File Server** (optional, for large payloads > 1MB) -3. **Julia** with required packages - ---- - -## Installation - -### Julia - -```julia -using Pkg -Pkg.add("NATS") -Pkg.add("Arrow") -Pkg.add("JSON3") -Pkg.add("HTTP") -Pkg.add("UUIDs") -Pkg.add("Dates") -``` - ---- - -## Quick Start - -### Step 1: Start NATS Server - -```bash -docker run -p 4222:4222 nats:latest -``` - -### Step 2: Start HTTP File Server (Optional) - -```bash -# Create a directory for file uploads -mkdir -p /tmp/fileserver - -# Use any HTTP server that supports POST for file uploads -python3 -m http.server 8080 --directory /tmp/fileserver -``` - -### Step 3: Send Your First Message - -#### Julia - -```julia -using NATSBridge - -# Send a text message -data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") -# env: msg_envelope_v1 object with all metadata and payloads -# env_json_str: JSON string representation of the envelope for publishing -println("Message sent!") - -# Or use is_publish=false to get envelope and JSON without publishing -env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=false) -# env: msg_envelope_v1 object -# env_json_str: JSON string for publishing to NATS -``` - -### Step 4: Receive Messages - -#### Julia - -```julia -using NATSBridge - -# Receive and process message -env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff) -# Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 -for (dataname, data, type) in env["payloads"] - println("Received $dataname: $data") -end -``` - ---- - -## Basic Examples - -### Example 1: Sending a Dictionary - -#### Julia - -```julia -using NATSBridge - -config = Dict( - "wifi_ssid" => "MyNetwork", - "wifi_password" => "password123", - "update_interval" => 60 -) - -data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222") -``` - -### Example 2: Sending Binary Data (Image) - -#### Julia - -```julia -using NATSBridge - -# Read image file -image_data = read("image.png") - -data = [("user_image", image_data, "binary")] -env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222") -``` - -### Example 3: Request-Response Pattern - -#### Julia (Requester) - -```julia -using NATSBridge - -# Send command with reply-to -data = [("command", Dict("action" => "read_sensor"), "dictionary")] -env, env_json_str = smartsend( - "/device/command", - data, - broker_url="nats://localhost:4222", - reply_to="/device/response", - reply_to_msg_id="cmd-001" -) -# env: msg_envelope_v1 object -# env_json_str: JSON string for publishing to NATS -``` - -#### Julia (Responder) - -```julia -using NATS, NATSBridge - -# Configuration -const SUBJECT = "/device/command" -const NATS_URL = "nats://localhost:4222" - -function test_responder() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) - - # Extract reply_to from the envelope metadata - reply_to = env["reply_to"] - - for (dataname, data, type) in env["payloads"] - if dataname == "command" && data["action"] == "read_sensor" - response = Dict("sensor_id" => "sensor-001", "value" => 42.5) - # Send response to the reply_to subject from the request - if !isempty(reply_to) - smartsend(reply_to, [("data", response, "dictionary")]) - end - end - end - end - - sleep(120) - NATS.drain(conn) -end - -test_responder() -``` - ---- - -## Advanced Usage - -### Example 4: Large Payloads (File Server) - -For payloads larger than 1MB, NATSBridge automatically uses the file server: - -#### Julia - -```julia -using NATSBridge - -# Create large data (> 1MB) -large_data = rand(UInt8, 2_000_000) - -env, env_json_str = smartsend( - "/data/large", - [("large_file", large_data, "binary")], - broker_url="nats://localhost:4222", - fileserver_url="http://localhost:8080" -) - -# The envelope will contain the download URL -println("File uploaded to: $(env.payloads[1].data)") -``` - -### Example 5: Mixed Content (Chat with Text + Image) - -NATSBridge supports sending multiple payloads with different types in a single message: - -#### Julia - -```julia -using NATSBridge - -image_data = read("avatar.png") - -data = [ - ("message_text", "Hello with image!", "text"), - ("user_avatar", image_data, "image") -] - -env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222") -``` - -### Example 6: Table Data (Arrow IPC) - -For tabular data, NATSBridge uses Apache Arrow IPC format: - -#### Julia - -```julia -using NATSBridge -using DataFrames - -# Create DataFrame -df = DataFrame( - id = [1, 2, 3], - name = ["Alice", "Bob", "Charlie"], - score = [95, 88, 92] -) - -data = [("students", df, "table")] -env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222") -``` - ---- - -## Next Steps - -1. **Explore the test directory** for more examples -2. **Check the documentation** for advanced configuration options - ---- - -## Troubleshooting - -### Connection Issues - -- Ensure NATS server is running: `docker ps | grep nats` -- Check firewall settings -- Verify NATS URL configuration - -### File Server Issues - -- Ensure file server is running and accessible -- Check upload permissions -- Verify file server URL configuration - -### Serialization Errors - -- Verify data type matches the specified type -- Check that binary data is in the correct format (Vector{UInt8}) - ---- - -## License - -MIT \ No newline at end of file diff --git a/examples/walkthrough.md b/examples/walkthrough.md deleted file mode 100644 index 995134f..0000000 --- a/examples/walkthrough.md +++ /dev/null @@ -1,706 +0,0 @@ -# NATSBridge Walkthrough - -A comprehensive guide to building real-world applications with NATSBridge. - -## Table of Contents - -1. [Introduction](#introduction) -2. [Architecture Overview](#architecture-overview) -3. [Building a Chat Application](#building-a-chat-application) -4. [Building a File Transfer System](#building-a-file-transfer-system) -5. [Building a Streaming Data Pipeline](#building-a-streaming-data-pipeline) -6. [Performance Optimization](#performance-optimimization) -7. [Best Practices](#best-practices) - ---- - -## Introduction - -This walkthrough will guide you through building several real-world applications using NATSBridge. We'll cover: - -- Chat applications with rich media support -- File transfer systems with claim-check pattern -- Streaming data pipelines - -Each section builds on the previous one, gradually increasing in complexity. - ---- - -## Architecture Overview - -### System Components - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ NATSBridge Architecture │ -├─────────────────────────────────────────────────────────────────┤ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ Julia │ │ NATS │ │ -│ │ (NATS.jl) │◄──►│ Server │ │ -│ └──────────────┘ └──────────────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌──────────────────────────────────────┐ │ -│ │ File Server │ │ -│ │ (HTTP Upload) │ │ -│ └──────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### Message Flow - -1. **Sender** creates a message envelope with payloads -2. **NATSBridge** serializes and encodes payloads -3. **Transport Decision**: Small payloads go directly to NATS, large payloads are uploaded to file server -4. **NATS** routes messages to subscribers -5. **Receiver** fetches payloads (from NATS or file server) -6. **NATSBridge** deserializes and decodes payloads - ---- - -## Building a Chat Application - -Let's build a full-featured chat application that supports text, images, and file attachments. - -### Step 1: Set Up the Project - -```bash -# Create project directory -mkdir -p chat-app/src -cd chat-app - -# Create configuration file -cat > config.json << 'EOF' -{ - "nats_url": "nats://localhost:4222", - "fileserver_url": "http://localhost:8080", - "size_threshold": 1048576 -} -EOF -``` - -### Step 2: Create the Chat Interface (Julia) - -```julia -# src/chat_ui.jl -using NATSBridge, NATS - -struct ChatUI - messages::Vector{Dict} - current_room::String -end - -function ChatUI() - ChatUI(Dict[], "") -end - -function send_message(ui::ChatUI, message_input::String, selected_file::Union{Nothing, String}) - data = [] - - # Add text message - if !isempty(message_input) - push!(data, ("text", message_input, "text")) - end - - # Add file if selected - if selected_file !== nothing - file_data = read(selected_file) - file_type = get_file_type(selected_file) - push!(data, ("attachment", file_data, file_type)) - end - - return data -end - -function get_file_type(filename::String)::String - if endswith(filename, ".png") || endswith(filename, ".jpg") - return "image" - elseif endswith(filename, ".mp3") || endswith(filename, ".wav") - return "audio" - elseif endswith(filename, ".mp4") || endswith(filename, ".avi") - return "video" - else - return "binary" - end -end - -function add_message(ui::ChatUI, user::String, text::String, attachment::Union{Nothing, Dict}) - push!(ui.messages, Dict( - "user" => user, - "text" => text, - "attachment" => attachment - )) -end -``` - -### Step 3: Create the Message Handler - -```julia -# src/chat_handler.jl -using NATSBridge, NATS - -struct ChatHandler - nats::NATS.Connection - ui::ChatUI -end - -function ChatHandler(nats_connection::NATS.Connection) - ChatHandler(nats_connection, ChatUI()) -end - -function start(handler::ChatHandler) - # Subscribe to chat rooms - rooms = ["general", "tech", "random"] - - for room in rooms - NATS.subscribe(handler.nats, "/chat/$room") do msg - handle_message(handler, msg) - end - end - - println("Chat handler started") -end - -function handle_message(handler::ChatHandler, msg::NATS.Msg) - env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) - # Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 - - # Extract sender info from envelope - sender = get(env, "sender_name", "Anonymous") - - # Process each payload - for (dataname, data, type) in env["payloads"] - if type == "text" - add_message(handler.ui, sender, data, nothing) - elseif type == "image" - # Convert to data URL for display - base64_data = base64encode(data) - attachment = Dict( - "type" => "image", - "data" => "data:image/png;base64,$base64_data" - ) - add_message(handler.ui, sender, "", attachment) - else - # For other types, use file server URL - attachment = Dict("type" => type, "data" => data) - add_message(handler.ui, sender, "", attachment) - end - end -end - -function download_file(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} - # Implement exponential backoff for file server downloads - # Return downloaded data as Vector{UInt8} -end -``` - -### Step 4: Run the Application - -```bash -# Start NATS -docker run -p 4222:4222 nats:latest - -# Start file server -mkdir -p /tmp/fileserver -python3 -m http.server 8080 --directory /tmp/fileserver - -# Run chat app -julia src/chat_ui.jl -julia src/chat_handler.jl -``` - ---- - -## Building a File Transfer System - -Let's build a file transfer system that handles large files efficiently. - -### Step 1: File Upload Service (Julia) - -```julia -# src/file_upload_service.jl -using NATSBridge, HTTP - -struct FileUploadService - broker_url::String - fileserver_url::String -end - -function FileUploadService(broker_url::String, fileserver_url::String) - FileUploadService(broker_url, fileserver_url) -end - -function upload_file(service::FileUploadService, file_path::String, recipient::String)::Dict - file_data = read(file_path) - file_name = basename(file_path) - - data = [("file", file_data, "binary")] - - env, env_json_str = smartsend( - "/files/$recipient", - data, - broker_url=service.broker_url, - fileserver_url=service.fileserver_url - ) - - return env -end - -function upload_large_file(service::FileUploadService, file_path::String, recipient::String)::Dict - file_size = stat(file_path).size - - if file_size > 100 * 1024 * 1024 # > 100MB - println("File too large for direct upload, using streaming...") - return stream_upload(service, file_path, recipient) - end - - return upload_file(service, file_path, recipient) -end - -function stream_upload(service::FileUploadService, file_path::String, recipient::String)::Dict - # Implement streaming upload to file server - # This would require a more sophisticated file server - # For now, we'll use the standard upload - return upload_file(service, file_path, recipient) -end -``` - -### Step 2: File Download Service (Julia) - -```julia -# src/file_download_service.jl -using NATSBridge - -struct FileDownloadService - nats_url::String -end - -function FileDownloadService(nats_url::String) - FileDownloadService(nats_url) -end - -function download_file(service::FileDownloadService, msg::NATS.Msg, sender::String, download_id::String) - # Subscribe to sender's file channel - env = smartreceive(msg, fileserver_download_handler=fetch_from_url) - # Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 - - # Process each payload - for (dataname, data, type) in env["payloads"] - if type == "binary" - file_path = "/downloads/$dataname" - write(file_path, data) - println("File saved to $file_path") - end - end -end - -function fetch_from_url(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} - # Fetch data from URL with exponential backoff - # Return downloaded data as Vector{UInt8} -end -``` - -### Step 3: File Transfer CLI (Julia) - -```julia -# src/cli.jl -using NATSBridge, Readlines, FileIO - -function main() - config = JSON3.read(read("config.json", String)) - - println("File Transfer System") - println("====================") - println("1. Upload file") - println("2. Download file") - println("3. List pending downloads") - - print("Enter choice: ") - choice = readline() - - if choice == "1" - upload_file_cli(config) - elseif choice == "2" - download_file_cli(config) - end -end - -function upload_file_cli(config) - print("Enter file path: ") - file_path = readline() - - print("Enter recipient: ") - recipient = readline() - - file_service = FileUploadService(config.nats_url, config.fileserver_url) - - try - env = upload_file(file_service, file_path, recipient) - println("Upload successful!") - println("File ID: $(env["payloads"][1][1])") - catch error - println("Upload failed: $(error)") - end -end - -function download_file_cli(config) - print("Enter sender: ") - sender = readline() - - file_service = FileDownloadService(config.nats_url) - - try - download_file(file_service, sender) - println("Download complete!") - catch error - println("Download failed: $(error)") - end -end - -main() -``` - ---- - -## Building a Streaming Data Pipeline - -Let's build a data pipeline that processes streaming data from sensors. - -### Step 1: Sensor Data Model (Julia) - -```julia -# src/sensor_data.jl -using Dates, DataFrames - -struct SensorReading - sensor_id::String - timestamp::String - value::Float64 - unit::String - metadata::Dict{String, Any} -end - -function SensorReading(sensor_id::String, value::Float64, unit::String, metadata::Dict{String, Any}=Dict()) - SensorReading( - sensor_id, - ISODateTime(now(), Dates.Second) |> string, - value, - unit, - metadata - ) -end - -struct SensorBatch - readings::Vector{SensorReading} -end - -function SensorBatch() - SensorBatch(SensorReading[]) -end - -function add_reading(batch::SensorBatch, reading::SensorReading) - push!(batch.readings, reading) -end - -function to_dataframe(batch::SensorBatch)::DataFrame - data = Dict{String, Any}() - data["sensor_id"] = [r.sensor_id for r in batch.readings] - data["timestamp"] = [r.timestamp for r in batch.readings] - data["value"] = [r.value for r in batch.readings] - data["unit"] = [r.unit for r in batch.readings] - - return DataFrame(data) -end -``` - -### Step 2: Sensor Sender (Julia) - -```julia -# src/sensor_sender.jl -using NATSBridge, Dates, Random - -struct SensorSender - broker_url::String - fileserver_url::String -end - -function SensorSender(broker_url::String, fileserver_url::String) - SensorSender(broker_url, fileserver_url) -end - -function send_reading(sender::SensorSender, sensor_id::String, value::Float64, unit::String) - reading = SensorReading(sensor_id, value, unit) - - data = [("reading", reading.metadata, "dictionary")] - - # Default: is_publish=True (automatically publishes to NATS) - smartsend( - "/sensors/$sensor_id", - data, - broker_url=sender.broker_url, - fileserver_url=sender.fileserver_url - ) -end - -function prepare_message_only(sender::SensorSender, sensor_id::String, value::Float64, unit::String) - """Prepare a message without publishing (is_publish=False).""" - reading = SensorReading(sensor_id, value, unit) - - data = [("reading", reading.metadata, "dictionary")] - - # With is_publish=False, returns (env, env_json_str) without publishing - env, env_json_str = smartsend( - "/sensors/$sensor_id/prepare", - data, - broker_url=sender.broker_url, - fileserver_url=sender.fileserver_url, - is_publish=false - ) - - # Now you can publish manually using NATS request-reply pattern - # nc.request(subject, env_json_str, reply_to=reply_to_topic) - - return env, env_json_str -end - -function send_batch(sender::SensorSender, readings::Vector{SensorReading}) - batch = SensorBatch() - for reading in readings - add_reading(batch, reading) - end - - df = to_dataframe(batch) - - # Convert to Arrow IPC format - import Arrow - table = Arrow.Table(df) - - # Serialize to Arrow IPC - import IOBuffer - buf = IOBuffer() - Arrow.write(buf, table) - - arrow_data = take!(buf) - - # Send based on size - if length(arrow_data) < 1048576 # < 1MB - data = [("batch", arrow_data, "table")] - smartsend( - "/sensors/batch", - data, - broker_url=sender.broker_url, - fileserver_url=sender.fileserver_url - ) - else - # Upload to file server - data = [("batch", arrow_data, "table")] - smartsend( - "/sensors/batch", - data, - broker_url=sender.broker_url, - fileserver_url=sender.fileserver_url - ) - end -end -``` - -### Step 3: Sensor Receiver (Julia) - -```julia -# src/sensor_receiver.jl -using NATSBridge, Arrow, DataFrames, IOBuffer - -struct SensorReceiver - fileserver_download_handler::Function -end - -function SensorReceiver(download_handler::Function) - SensorReceiver(download_handler) -end - -function process_reading(receiver::SensorReceiver, msg::NATS.Msg) - env = smartreceive(msg, receiver.fileserver_download_handler) - # Returns: ::JSON.Object{String, Any} - key-value structure resemble msg_envelope_v1 - - for (dataname, data, data_type) in env["payloads"] - if data_type == "dictionary" - # Process dictionary payload - println("Received: $dataname = $data") - elseif data_type == "table" - # Deserialize Arrow IPC - buf = IOBuffer(data) - table = Arrow.read(buf) - df = DataFrame(table) - println("Received batch with $(nrow(df)) readings") - println(df) - end - end -end -``` - ---- - -## Performance Optimization - -### 1. Batch Processing - -```julia -# Batch multiple readings into a single message -function send_batch_readings(sender::SensorSender, readings::Vector{Tuple{String, Float64, String}}) - batch = SensorBatch() - - for (sensor_id, value, unit) in readings - reading = SensorReading(sensor_id, value, unit) - add_reading(batch, reading) - end - - df = to_dataframe(batch) - - # Convert to Arrow IPC - import Arrow - table = Arrow.Table(df) - - # Serialize to Arrow IPC - import IOBuffer - buf = IOBuffer() - Arrow.write(buf, table) - - arrow_data = take!(buf) - - # Send as single message - smartsend( - "/sensors/batch", - [("batch", arrow_data, "table")], - broker_url=sender.broker_url - ) -end -``` - -### 2. Connection Reuse - -```julia -# Reuse NATS connections -function create_connection_pool() - connections = Dict{String, NATS.Connection}() - - function get_connection(nats_url::String)::NATS.Connection - if !haskey(connections, nats_url) - connections[nats_url] = NATS.connect(nats_url) - end - return connections[nats_url] - end - - function close_all() - for conn in values(connections) - NATS.drain(conn) - end - empty!(connections) - end - - return (get_connection= get_connection, close_all=close_all) -end -``` - -### 3. Caching - -```julia -# Cache file server responses -using Base.Threads - -const file_cache = Dict{String, Vector{UInt8}}() - -function fetch_with_caching(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} - if haskey(file_cache, url) - return file_cache[url] - end - - # Fetch from file server - data = _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) - - # Cache the result - file_cache[url] = data - - return data -end -``` - ---- - -## Best Practices - -### 1. Error Handling - -```julia -function safe_smartsend(subject::String, data::Vector{Tuple}, kwargs...) - try - return smartsend(subject, data; kwargs...) - catch error - println("Failed to send message: $(error)") - return nothing - end -end -``` - -### 2. Logging - -```julia -using Logging - -function log_send(subject::String, data::Vector{Tuple}, correlation_id::String) - @info "Sending to $subject: $(length(data)) payloads, correlation_id=$correlation_id" -end - -function log_receive(correlation_id::String, num_payloads::Int) - @info "Received message: $num_payloads payloads, correlation_id=$correlation_id" -end -``` - -### 3. Rate Limiting - -```julia -using Dates, Collections - -struct RateLimiter - max_requests::Int - time_window::Float64 - requests::Deque{Float64} -end - -function RateLimiter(max_requests::Int, time_window::Float64) - RateLimiter(max_requests, time_window, Deque{Float64}()) -end - -function allow(limiter::RateLimiter)::Bool - now = time() - - # Remove old requests - while !isempty(limiter.requests) && limiter.requests[1] < now - limiter.time_window - popfirst!(limiter.requests) - end - - if length(limiter.requests) >= limiter.max_requests - return false - end - - push!(limiter.requests, now) - return true -end -``` - ---- - -## Conclusion - -This walkthrough covered: - -- Building a chat application with rich media support -- Building a file transfer system with claim-check pattern -- Building a streaming data pipeline for sensor data - -For more information, check the [API documentation](../src/README.md) and [test examples](../test/). - ---- - -## License - -MIT \ No newline at end of file diff --git a/package.json b/package.json deleted file mode 100644 index 1b5ccdc..0000000 --- a/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "natsbridge", - "version": "1.0.0", - "description": "Bi-Directional Data Bridge for JavaScript using NATS", - "main": "src/NATSBridge.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", - "lint": "eslint src/*.js test/*.js" - }, - "keywords": [ - "nats", - "message-broker", - "bridge", - "arrow", - "serialization" - ], - "author": "", - "license": "MIT", - "dependencies": { - "nats": "^2.9.0", - "apache-arrow": "^14.0.0", - "uuid": "^9.0.0" - }, - "devDependencies": { - "eslint": "^8.0.0", - "jest": "^29.0.0" - } -} \ No newline at end of file diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index e53d388..b0f781e 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -31,11 +31,19 @@ # [(dataname1, data1, type1), (dataname2, data2, type2), ...] # ``` # -# Supported types: "text", "dictionary", "table", "image", "audio", "video", "binary" +# Supported types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" +# +# Table Datatypes: +# - `arrowtable`: Apache Arrow IPC format for efficient binary serialization +# - Input: DataFrame, Arrow.Table +# - Encoding: arrow-ipc +# - `jsontable`: JSON format for human-readable tabular data +# - Input: Vector{NamedTuple}, Vector{Dict} +# - Encoding: json module NATSBridge -using NATS, JSON, Arrow, HTTP, UUIDs, Dates, Base64, PrettyPrinting +using NATS, JSON, Arrow, HTTP, UUIDs, Dates, Base64, PrettyPrinting, DataFrames # ---------------------------------------------- 100 --------------------------------------------- # # Constants @@ -51,7 +59,7 @@ It supports both direct transport (base64-encoded data) and link transport (URL- # Arguments: - `id::String` - Unique identifier for this payload (e.g., "uuid4") - `dataname::String` - Name of the payload (e.g., "login_image") - - `payload_type::String` - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" + - `payload_type::String` - Payload type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" - `transport::String` - Transport method: "direct" or "link" - `encoding::String` - Encoding method: "none", "json", "base64", "arrow-ipc" - `size::Integer` - Size of the payload in bytes (e.g., 15433) @@ -100,7 +108,7 @@ payload = msg_payload_v1( struct msg_payload_v1 id::String # id of this payload e.g. "uuid4" dataname::String # name of this payload e.g. "login_image" - payload_type::String # this payload type. Can be "text", "dictionary", "table", "image", "audio", "video", "binary" + payload_type::String # this payload type. Can be "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" transport::String # transport method: "direct" or "link" encoding::String # encoding method: "none", "json", "base64", "arrow-ipc" size::Integer # data size in bytes e.g. 15433 @@ -279,42 +287,26 @@ function envelope_to_json(env::msg_envelope_v1) "broker_url" => env.broker_url ) - if !isempty(env.metadata) # Only include metadata if it exists and is not empty - obj["metadata"] = Dict(String(k) => v for (k, v) in env.metadata) - end + obj["metadata"] = Dict(String(k) => v for (k, v) in env.metadata) # Convert payloads to JSON array - if !isempty(env.payloads) - payloads_json = [] - for payload in env.payloads - payload_obj = Dict{String, Any}( - "id" => payload.id, - "dataname" => payload.dataname, - "payload_type" => payload.payload_type, - "transport" => payload.transport, - "encoding" => payload.encoding, - "size" => payload.size, - ) - # Include data based on transport type - if payload.transport == "direct" && payload.data !== nothing - if payload.encoding == "base64" || payload.encoding == "json" - payload_obj["data"] = payload.data - else - # For other encodings, use base64 - payload_bytes = _get_payload_bytes(payload.data) - payload_obj["data"] = Base64.base64encode(payload_bytes) - end - elseif payload.transport == "link" && payload.data !== nothing - # For link transport, data is a URL string - include directly - payload_obj["data"] = payload.data - end - if !isempty(payload.metadata) - payload_obj["metadata"] = Dict(String(k) => v for (k, v) in payload.metadata) - end - push!(payloads_json, payload_obj) + payloads_json = [] + for payload in env.payloads + payload_obj = Dict{String, Any}( + "id" => payload.id, + "dataname" => payload.dataname, + "payload_type" => payload.payload_type, + "transport" => payload.transport, + "encoding" => payload.encoding, + "size" => payload.size, + ) + payload_obj["data"] = payload.data + if !isempty(payload.metadata) + payload_obj["metadata"] = Dict(String(k) => v for (k, v) in payload.metadata) end - obj["payloads"] = payloads_json + push!(payloads_json, payload_obj) end + obj["payloads"] = payloads_json JSON.json(obj) end @@ -367,7 +359,7 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c - `data::AbstractArray{Tuple{String, Any, String}}` - List of (dataname, data, type) tuples to send - `dataname::String` - Name of the payload - `data::Any` - The actual data to send - - `payload_type::String` - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" + - `payload_type::String` - Payload type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" - No standalone `type` parameter - type is specified per payload # Keyword Arguments: @@ -403,11 +395,15 @@ env, msg_json = smartsend("my.subject", [("dataname1", data, "dictionary")]) # Send multiple payloads in one message with different types data1 = Dict("key1" => "value1") data2 = rand(10_000) # Small array -env, msg_json = smartsend("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")]) +env, msg_json = smartsend("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")]) # Send a large array using fileserver upload data = rand(10_000_000) # ~80 MB -env, msg_json = smartsend("large.data", [("large_table", data, "table")]) +env, msg_json = smartsend("large.data", [("large_arrow_table", data, "arrowtable")]) + +# Send jsontable (JSON format) +rows = [Dict("id" => 1, "name" => "Alice"), Dict("id" => 2, "name" => "Bob")] +env, msg_json = smartsend("json.data", [("users", rows, "jsontable")]) # Mixed content (e.g., chat with text and image) env, msg_json = smartsend("chat.subject", [ @@ -428,13 +424,12 @@ function smartsend( fileserver_upload_handler::Function = plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver size_threshold::Int = DEFAULT_SIZE_THRESHOLD, - #= - Generate a globally unique identifier (UUID) at the start of the request. - This ID must remain constant and immutable as it propagates through every - stage of the execution pipeline. It serves as the end-to-end ID for - distributed tracing, enabling the correlation of all logs, metrics, and - errors across the system back to this specific request instance. - =# + # Generate a globally unique identifier (UUID) at the start of the request. + # This ID must remain constant and immutable as it propagates through every + # stage of the execution pipeline. It serves as the end-to-end ID for + # distributed tracing, enabling the correlation of all logs, metrics, and + # errors across the system back to this specific request instance. + correlation_id::String = string(uuid4()), msg_purpose::String = "chat", @@ -455,6 +450,8 @@ function smartsend( # Process each payload in the list payloads = msg_payload_v1[] for (dataname, payload_data, payload_type) in data + @show dataname typeof(payload_data) + # Serialize data based on type payload_bytes = _serialize_data(payload_data, payload_type) @@ -467,6 +464,14 @@ function smartsend( payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string log_trace(correlation_id, "Using direct transport for $payload_size bytes") # Log transport choice + # Determine encoding based on payload_type + encoding = "base64" + if payload_type == "jsontable" + encoding = "json" + elseif payload_type == "arrowtable" + encoding = "arrow-ipc" + end + # Create msg_payload_v1 for direct transport payload = msg_payload_v1( payload_b64, @@ -474,7 +479,7 @@ function smartsend( id = string(uuid4()), dataname = dataname, transport = "direct", - encoding = "base64", + encoding = encoding, size = payload_size, metadata = Dict{String, Any}("payload_bytes" => payload_size) ) @@ -485,7 +490,7 @@ function smartsend( # Upload to HTTP server response = fileserver_upload_handler(fileserver_url, dataname, payload_bytes) - + if response["status"] != 200 # Check if upload was successful error("Failed to upload data to fileserver: $(response["status"])") # Throw error if upload failed end @@ -493,6 +498,14 @@ function smartsend( url = response["url"] # URL for the uploaded data log_trace(correlation_id, "Uploaded to URL: $url") # Log successful upload + # Determine encoding based on payload_type + encoding = "none" + if payload_type == "jsontable" + encoding = "json" + elseif payload_type == "arrowtable" + encoding = "arrow-ipc" + end + # Create msg_payload_v1 for link transport payload = msg_payload_v1( url, @@ -500,7 +513,7 @@ function smartsend( id = string(uuid4()), dataname = dataname, transport = "link", - encoding = "none", + encoding = encoding, size = payload_size, metadata = Dict{String, Any}() ) @@ -547,12 +560,13 @@ It supports multiple serialization formats for different data types. 2. Converts data to binary representation according to format rules 3. For text: converts string to UTF-8 bytes 4. For dictionary: serializes as JSON then converts to bytes -5. For table: uses Arrow.jl to write as IPC stream -6. For image/audio/video/binary: returns binary data directly +5. For arrowtable: uses Arrow.jl to write as IPC stream +6. For jsontable: converts to JSON then to bytes +7. For image/audio/video/binary: returns binary data directly # Arguments: - - `data::Any` - Data to serialize (string for `"text"`, JSON-serializable for `"dictionary"`, table-like for `"table"`, binary for `"image"`, `"audio"`, `"video"`, `"binary"`) - - `payload_type::String` - Target format: "text", "dictionary", "table", "image", "audio", "video", "binary" + - `data::Any` - Data to serialize (string for `"text"`, JSON-serializable for `"dictionary"`, table-like for `"arrowtable"`, Vector{NamedTuple}/Vector{Dict} for `"jsontable"`, binary for `"image"`, `"audio"`, `"video"`, `"binary"`) + - `payload_type::String` - Target format: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" # Return: - `Vector{UInt8}` - Binary representation of the serialized data @@ -573,9 +587,13 @@ text_bytes = _serialize_data(text_data, "text") json_data = Dict("name" => "Alice", "age" => 30) json_bytes = _serialize_data(json_data, "dictionary") -# Table serialization with a DataFrame (recommended for tabular data) +# Arrow table serialization with a DataFrame (recommended for tabular data) df = DataFrame(id = 1:3, name = ["Alice", "Bob", "Charlie"], score = [95, 88, 92]) -table_bytes = _serialize_data(df, "table") +arrow_bytes = _serialize_data(df, "arrowtable") + +# JSON table serialization - Vector{NamedTuple} or Vector{Dict} +rows = [Dict("id" => 1, "name" => "Alice"), Dict("id" => 2, "name" => "Bob")] +json_bytes = _serialize_data(rows, "jsontable") # Image data (Vector{UInt8}) image_bytes = UInt8[1, 2, 3] # Image bytes @@ -626,10 +644,30 @@ function _serialize_data(data::Any, payload_type::String) json_str = JSON.json(data) # Convert Julia data to JSON string json_str_bytes = Vector{UInt8}(json_str) # Convert JSON string to bytes return json_str_bytes - elseif payload_type == "table" # Table data - convert to Arrow IPC stream + elseif payload_type == "arrowtable" # Arrow table data - convert to Arrow IPC stream io = IOBuffer() # Create in-memory buffer Arrow.write(io, data) # Write data as Arrow IPC stream to buffer return take!(io) # Return the buffer contents as bytes + elseif payload_type == "jsontable" # JSON table data - convert to JSON + # data can be Vector{NamedTuple}, Vector{Dict}, or DataFrame + # If DataFrame, convert to Vector{Dict} first + if isa(data, DataFrame) + # Convert DataFrame to Vector{Dict} (row-oriented) + rows = [] + for i in 1:nrow(data) + row_dict = Dict() + for col in names(data) + row_dict[String(col)] = data[i, col] + end + push!(rows, row_dict) + end + json_str = JSON.json(rows) + return Vector{UInt8}(json_str) + else + # Already Vector{NamedTuple} or Vector{Dict} + json_str = JSON.json(data) + return Vector{UInt8}(json_str) + end elseif payload_type == "image" # Image data - treat as binary if isa(data, Vector{UInt8}) return data # Return binary data directly @@ -885,24 +923,25 @@ end """ _deserialize_data - Deserialize bytes to data based on type This internal function converts serialized bytes back to Julia data based on type. -It handles "text" (string), "dictionary" (JSON deserialization), "table" (Arrow IPC deserialization), -"image" (binary data), "audio" (binary data), "video" (binary data), and "binary" (binary data). +It handles "text" (string), "dictionary" (JSON deserialization), "arrowtable" (Arrow IPC deserialization), +"jsontable" (JSON deserialization), "image" (binary data), "audio" (binary data), "video" (binary data), and "binary" (binary data). # Function Workflow: 1. Validates the data type against supported formats 2. Converts bytes to appropriate Julia data type based on format 3. For text: converts bytes to string 4. For dictionary: converts bytes to JSON string then parses to Julia object -5. For table: reads Arrow IPC format and returns DataFrame -6. For image/audio/video/binary: returns bytes directly +5. For arrowtable: reads Arrow IPC format and returns Arrow.Table +6. For jsontable: converts bytes to JSON string then parses to Vector{Dict} +7. For image/audio/video/binary: returns bytes directly # Arguments: - `data::Vector{UInt8}` - Serialized data as bytes - - `payload_type::String` - Data type ("text", "dictionary", "table", "image", "audio", "video", "binary") + - `payload_type::String` - Data type ("text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary") - `correlation_id::String` - Correlation ID for logging # Return: - - Deserialized data (String for "text", DataFrame for "table", JSON data for "dictionary", bytes for "image", "audio", "video", "binary") + - Deserialized data (String for "text", Arrow.Table for "arrowtable", Vector{Dict} for "jsontable", JSON data for "dictionary", bytes for "image", "audio", "video", "binary") # Throws: - `Error` if `payload_type` is not one of the supported types @@ -917,9 +956,13 @@ text_data = _deserialize_data(text_bytes, "text", "correlation123") json_bytes = UInt8[123, 34, 110, 97, 109, 101, 34, 58, 34, 65, 108, 105, 99, 101, 125] # {"name":"Alice"} json_data = _deserialize_data(json_bytes, "dictionary", "correlation123") -# Arrow IPC data (table) +# Arrow IPC data (arrowtable) arrow_bytes = Vector{UInt8}([1, 2, 3]) # Arrow IPC bytes -table_data = _deserialize_data(arrow_bytes, "table", "correlation123") +arrow_table = _deserialize_data(arrow_bytes, "arrowtable", "correlation123") + +# JSON table data (jsontable) +json_table_bytes = UInt8[91, 123, 34, 105, 100, 34, 58, 49, 44, 34, 110, 97, 109, 101, 34, 58, 34, 65, 108, 105, 99, 101, 34, 125] # [{"id":1,"name":"Alice"}] +json_table = _deserialize_data(json_table_bytes, "jsontable", "correlation123") ``` """ function _deserialize_data( @@ -932,10 +975,13 @@ function _deserialize_data( elseif payload_type == "dictionary" # JSON data - deserialize json_str = String(data) # Convert bytes to string return JSON.parse(json_str) # Parse JSON string to JSON object - elseif payload_type == "table" # Table data - deserialize Arrow IPC stream + elseif payload_type == "arrowtable" # Arrow table data - deserialize Arrow IPC stream io = IOBuffer(data) # Create buffer from bytes - df = Arrow.Table(io) # Read Arrow IPC format from buffer - return df # Return DataFrame + table = Arrow.Table(io) # Read Arrow IPC format from buffer + return table # Return Arrow.Table + elseif payload_type == "jsontable" # JSON table data - deserialize JSON + json_str = String(data) # Convert bytes to string + return JSON.parse(json_str) # Parse JSON string to Vector{Dict} elseif payload_type == "image" # Image data - return binary return data # Return bytes directly elseif payload_type == "audio" # Audio data - return binary @@ -974,19 +1020,19 @@ retrieves an upload ID and token, then uploads the file data as multipart form d - `"url"` - Full URL to download the uploaded file # Example - ```jldoctest - using HTTP, JSON + ```jldoctest + using HTTP, JSON - fileserver_url = "http://localhost:8080" - dataname = "test.txt" - data = Vector{UInt8}("hello world") + fileserver_url = "http://localhost:8080" + dataname = "test.txt" + data = Vector{UInt8}("hello world") - # Upload to local plik server - result = plik_oneshot_upload(file_server_url, dataname, data) + # Upload to local plik server + result = plik_oneshot_upload(file_server_url, dataname, data) - # Access the result as a Dict - # result["status"], result["uploadid"], result["fileid"], result["url"] - ``` + # Access the result as a Dict + # result["status"], result["uploadid"], result["fileid"], result["url"] + ``` """ function plik_oneshot_upload(file_server_url::String, dataname::String, data::Vector{UInt8}) @@ -1110,18 +1156,4 @@ end - - - - - - - - - - - - - - end # module diff --git a/src/natsbridge.js b/src/natsbridge.js new file mode 100644 index 0000000..cb83e92 --- /dev/null +++ b/src/natsbridge.js @@ -0,0 +1,788 @@ +/** + * NATSBridge - Cross-Platform Bi-Directional Data Bridge + * JavaScript/Node.js Implementation + * + * This module provides functionality for sending and receiving data across network boundaries + * using NATS as the message bus, with support for both direct payload transport and + * URL-based transport for larger payloads. + * + * Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" + * + * @module NATSBridge + */ + +const nats = require('nats'); +const crypto = require('crypto'); +// Use native fetch available in Node.js 18+ +const arrow = require('apache-arrow'); + +// ---------------------------------------------- Constants ---------------------------------------------- // + +/** + * Default size threshold for switching from direct to link transport (1MB) + */ +const DEFAULT_SIZE_THRESHOLD = 1_000_000; + +/** + * Default NATS server URL + */ +const DEFAULT_BROKER_URL = 'nats://localhost:4222'; + +/** + * Default HTTP file server URL for link transport + */ +const DEFAULT_FILESERVER_URL = 'http://localhost:8080'; + +// ---------------------------------------------- Utility Functions ---------------------------------------------- // + +/** + * Convert Buffer to Base64 string + * @param {Buffer} buffer - Buffer to encode + * @returns {string} Base64 encoded string + */ +function bufferToBase64(buffer) { + return buffer.toString('base64'); +} + +/** + * Log a trace message with correlation ID and timestamp + * @param {string} correlationId - Correlation ID for tracing + * @param {string} message - Message content to log + */ +function logTrace(correlationId, message) { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] [Correlation: ${correlationId}] ${message}`); +} + +// ---------------------------------------------- Serialization Functions ---------------------------------------------- // + +/** + * Serialize data according to specified format + * @param {any} data - Data to serialize + * @param {string} payloadType - Target format: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" + * @returns {Buffer} Binary representation of the serialized data + */ +async function serializeData(data, payloadType) { + if (payloadType === 'text') { + if (typeof data === 'string') { + return Buffer.from(data, 'utf8'); + } else { + throw new Error('Text data must be a string'); + } + } else if (payloadType === 'dictionary') { + const jsonStr = JSON.stringify(data); + return Buffer.from(jsonStr, 'utf8'); + } else if (payloadType === 'arrowtable') { + // Convert array of objects to Arrow IPC format + if (!Array.isArray(data) || data.length === 0) { + throw new Error('Arrow table data must be a non-empty array of objects'); + } + + return serializeArrowTable(data); + } else if (payloadType === 'jsontable') { + // Serialize array of objects to JSON format + if (!Array.isArray(data)) { + throw new Error('JSON table data must be an array'); + } + const jsonStr = JSON.stringify(data); + return Buffer.from(jsonStr, 'utf8'); + } else if (payloadType === 'image') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Image data must be Uint8Array or Buffer'); + } + } else if (payloadType === 'audio') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Audio data must be Uint8Array or Buffer'); + } + } else if (payloadType === 'video') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Video data must be Uint8Array or Buffer'); + } + } else if (payloadType === 'binary') { + if (data instanceof Uint8Array || Buffer.isBuffer(data)) { + return Buffer.from(data); + } else { + throw new Error('Binary data must be Uint8Array or Buffer'); + } + } else { + throw new Error(`Unknown payload_type: ${payloadType}`); + } +} + +/** + * Helper function to properly serialize table data to Arrow IPC + * @param {Array} data - Array of objects representing table rows + * @returns {Buffer} Arrow IPC formatted buffer + */ +function serializeArrowTable(data) { + if (!Array.isArray(data) || data.length === 0) { + throw new Error('Table data must be a non-empty array of objects'); + } + + logTrace('serializeArrowTable', `Serializing table with ${data.length} rows`); + + // Use arrow.tableFromArrays which handles the conversion properly + // Convert array of objects to a key-value format expected by tableFromArrays + const columns = {}; + for (const key of Object.keys(data[0])) { + columns[key] = data.map(row => row[key]); + } + + logTrace('serializeArrowTable', `Columns: ${Object.keys(columns).join(', ')}`); + + const table = arrow.tableFromArrays(columns); + + logTrace('serializeArrowTable', `Arrow table created with ${table.numRows} rows, ${table.numCols} cols`); + + // Convert to IPC format + const ipcBuffer = arrow.tableToIPC(table); + + logTrace('serializeArrowTable', `IPC buffer type: ${typeof ipcBuffer}, length: ${ipcBuffer.byteLength}`); + + const resultBuffer = Buffer.from(ipcBuffer); + logTrace('serializeArrowTable', `Result buffer: ${resultBuffer.length} bytes`); + + // Debug: Show first 20 bytes in hex + const hexPreview = resultBuffer.slice(0, 20).toString('hex'); + logTrace('serializeArrowTable', `First 20 bytes (hex): ${hexPreview}`); + + return resultBuffer; +} + +/** + * Deserialize bytes to data based on type + * @param {Buffer|Uint8Array} data - Serialized data as bytes + * @param {string} payloadType - Data type + * @param {string} correlationId - Correlation ID for logging + * @returns {any} Deserialized data + */ +async function deserializeData(data, payloadType, correlationId) { + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + + logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`); + + // Debug: Show first 20 bytes in hex for binary data + if (payloadType === 'arrowtable' || payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') { + const hexPreview = buffer.slice(0, 20).toString('hex'); + logTrace(correlationId, `deserializeData: First 20 bytes (hex): ${hexPreview}`); + } + + if (payloadType === 'text') { + const result = buffer.toString('utf8'); + logTrace(correlationId, `deserializeData: text result length=${result.length}`); + return result; + } else if (payloadType === 'dictionary') { + const jsonStr = buffer.toString('utf8'); + const result = JSON.parse(jsonStr); + logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`); + return result; + } else if (payloadType === 'arrowtable') { + logTrace(correlationId, `deserializeData: Attempting Arrow table deserialization`); + + // Debug: Check available arrow methods + logTrace(correlationId, `deserializeData: arrow.tableFromRawBytes exists: ${typeof arrow.tableFromRawBytes}`); + logTrace(correlationId, `deserializeData: arrow.tableFromIPC exists: ${typeof arrow.tableFromIPC}`); + + try { + // Try tableFromRawBytes first (older API) + if (typeof arrow.tableFromRawBytes === 'function') { + logTrace(correlationId, `deserializeData: Using tableFromRawBytes`); + const table = arrow.tableFromRawBytes(buffer); + logTrace(correlationId, `deserializeData: Arrow table - rows=${table.numRows}, cols=${table.numCols}`); + return table; + } + } catch (e) { + logTrace(correlationId, `deserializeData: tableFromRawBytes failed: ${e.message}`); + } + + try { + // Try tableFromIPC (newer API) + if (typeof arrow.tableFromIPC === 'function') { + logTrace(correlationId, `deserializeData: Using tableFromIPC`); + const table = arrow.tableFromIPC(buffer); + logTrace(correlationId, `deserializeData: Arrow table from IPC - rows=${table.numRows}, cols=${table.numCols}`); + return table; + } + } catch (e) { + logTrace(correlationId, `deserializeData: tableFromIPC failed: ${e.message}`); + } + + throw new Error(`Unable to deserialize Arrow table: neither tableFromRawBytes nor tableFromIPC worked`); + } else if (payloadType === 'jsontable') { + const jsonStr = buffer.toString('utf8'); + const result = JSON.parse(jsonStr); + logTrace(correlationId, `deserializeData: jsontable result length=${Array.isArray(result) ? result.length : 'N/A'}`); + return result; + } else if (payloadType === 'image') { + logTrace(correlationId, `deserializeData: image buffer length=${buffer.length}`); + return buffer; + } else if (payloadType === 'audio') { + logTrace(correlationId, `deserializeData: audio buffer length=${buffer.length}`); + return buffer; + } else if (payloadType === 'video') { + logTrace(correlationId, `deserializeData: video buffer length=${buffer.length}`); + return buffer; + } else if (payloadType === 'binary') { + logTrace(correlationId, `deserializeData: binary buffer length=${buffer.length}`); + return buffer; + } else { + throw new Error(`Unknown payload_type: ${payloadType}`); + } +} + +// ---------------------------------------------- File Server Handlers ---------------------------------------------- // + +/** + * Upload data to plik server in one-shot mode + * @param {string} fileServerUrl - Base URL of the plik server + * @param {string} dataname - Name of the file being uploaded + * @param {Buffer|Uint8Array} data - Raw byte data of the file content + * @returns {Promise<{status: number, uploadid: string, fileid: string, url: string}>} + */ +async function plikOneshotUpload(fileServerUrl, dataname, data) { + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + + // Get upload id + const urlGetUploadID = `${fileServerUrl}/upload`; + const headers = { 'Content-Type': 'application/json' }; + const body = JSON.stringify({ OneShot: true }); + + const httpResponse = await fetch(urlGetUploadID, { + method: 'POST', + headers, + body + }); + + const responseJson = await httpResponse.json(); + const uploadid = responseJson.id; + const uploadtoken = responseJson.uploadToken; + + // Upload file + const urlUpload = `${fileServerUrl}/file/${uploadid}`; + const form = new FormData(); + const blob = new Blob([buffer], { type: 'application/octet-stream' }); + form.append('file', blob, dataname); + + const uploadHeaders = { + 'X-UploadToken': uploadtoken + }; + + const uploadResponse = await fetch(urlUpload, { + method: 'POST', + headers: uploadHeaders, + body: form + }); + + const uploadJson = await uploadResponse.json(); + const fileid = uploadJson.id; + + const url = `${fileServerUrl}/file/${uploadid}/${fileid}/${dataname}`; + + return { + status: uploadResponse.status, + uploadid, + fileid, + url + }; +} + +/** + * Fetch data from URL with exponential backoff + * @param {string} url - URL to fetch from + * @param {number} maxRetries - Maximum number of retry attempts + * @param {number} baseDelay - Initial delay in milliseconds + * @param {number} maxDelay - Maximum delay in milliseconds + * @param {string} correlationId - Correlation ID for logging + * @returns {Promise} Fetched data as bytes + */ +async function fetchWithBackoff(url, maxRetries, baseDelay, maxDelay, correlationId) { + let delay = baseDelay; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const response = await fetch(url); + + if (response.status === 200) { + logTrace(correlationId, `Successfully fetched data from ${url} on attempt ${attempt}`); + const arrayBuffer = await response.arrayBuffer(); + return new Uint8Array(arrayBuffer); + } else { + throw new Error(`Failed to fetch: ${response.status}`); + } + } catch (e) { + logTrace(correlationId, `Attempt ${attempt} failed: ${e.constructor.name} - ${e.message}`); + + if (attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, delay)); + delay = Math.min(delay * 2, maxDelay); + } + } + } + + throw new Error(`Failed to fetch data after ${maxRetries} attempts`); +} + +// ---------------------------------------------- NATS Client ---------------------------------------------- // + +/** + * NATS client wrapper for connection management + */ +class NATSClient { + /** + * Create a new NATS client + * @param {string} url - NATS server URL + */ + constructor(url) { + this.url = url; + this.connection = null; + } + + /** + * Connect to NATS server + * @returns {Promise} + */ + async connect() { + this.connection = await nats.connect({ servers: this.url }); + return this.connection; + } + + /** + * Publish message to NATS subject + * @param {string} subject - NATS subject to publish to + * @param {string} message - Message to publish + * @param {string} correlationId - Correlation ID for logging + */ + async publish(subject, message, correlationId) { + if (!this.connection) { + await this.connect(); + } + await this.connection.publish(subject, message); + logTrace(correlationId, `Message published to ${subject}`); + } + + /** + * Close the NATS connection + */ + async close() { + if (this.connection) { + this.connection.close(); + } + } +} + +// ---------------------------------------------- Core Functions ---------------------------------------------- // + +/** + * Publish message to NATS + * @param {string|NATSClient|NATS.Connection} brokerUrlOrClient - NATS URL, client, or connection + * @param {string} subject - NATS subject to publish to + * @param {string} message - JSON message to publish + * @param {string} correlationId - Correlation ID for tracing + */ +async function publishMessage(brokerUrlOrClient, subject, message, correlationId) { + let conn; + + if (brokerUrlOrClient instanceof NATSClient) { + conn = brokerUrlOrClient; + } else if (brokerUrlOrClient && typeof brokerUrlOrClient.publish === 'function') { + // Create a wrapper for direct connection (duck-typing check for NATS connection) + conn = { + async publish(subj, msg) { + await brokerUrlOrClient.publish(subj, msg); + }, + async close() { + await brokerUrlOrClient.close(); + } + }; + } else { + // String URL - create new client + const client = new NATSClient(brokerUrlOrClient); + conn = client; + } + + await conn.publish(subject, message, correlationId); + + if (conn instanceof NATSClient) { + await conn.close(); + } +} + +/** + * Build message envelope from payloads and metadata + * @param {string} subject - NATS subject + * @param {Array} payloads - Array of payload objects + * @param {Object} options - Envelope metadata options + * @returns {Object} Envelope object + */ +function buildEnvelope(subject, payloads, options) { + return { + correlation_id: options.correlation_id, + msg_id: options.msg_id, + timestamp: new Date().toISOString(), + send_to: subject, + msg_purpose: options.msg_purpose, + sender_name: options.sender_name, + sender_id: options.sender_id, + receiver_name: options.receiver_name, + receiver_id: options.receiver_id, + reply_to: options.reply_to, + reply_to_msg_id: options.reply_to_msg_id, + broker_url: options.broker_url, + metadata: options.metadata || {}, + payloads: payloads + }; +} + +/** + * Build payload object from serialized data + * @param {string} dataname - Name of the payload + * @param {string} payloadType - Type of the payload + * @param {Buffer} payloadBytes - Serialized payload bytes + * @param {string} transport - Transport type ("direct" or "link") + * @param {string} data - Data (base64 for direct, URL for link) + * @returns {Object} Payload object + */ +function buildPayload(dataname, payloadType, payloadBytes, transport, data) { + // Determine encoding based on payload type (matching Julia implementation) + let encoding = 'base64'; + if (payloadType === 'jsontable') { + encoding = 'json'; + } else if (payloadType === 'arrowtable') { + encoding = 'arrow-ipc'; + } + + return { + id: crypto.randomUUID(), + dataname, + payload_type: payloadType, + transport, + encoding, + size: payloadBytes.byteLength, + data, + metadata: transport === 'direct' ? { payload_bytes: payloadBytes.byteLength } : {} + }; +} + +/** + * Send data via NATS with automatic transport selection + * + * This function intelligently routes data delivery based on payload size. + * If the serialized payload is smaller than size_threshold, it encodes the data as Base64 + * and publishes directly over NATS. Otherwise, it uploads the data to a fileserver + * and publishes only the download URL over NATS. + * + * @param {string} subject - NATS subject to publish the message to + * @param {Array} data - List of [dataname, data, type] tuples to send + * - type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" + * @param {Object} options - Optional configuration + * @param {string} [options.broker_url=DEFAULT_BROKER_URL] - URL of the NATS server + * @param {string} [options.fileserver_url=DEFAULT_FILESERVER_URL] - URL of the HTTP file server + * @param {Function} [options.fileserver_upload_handler=plikOneshotUpload] - Function to handle fileserver uploads + * @param {number} [options.size_threshold=DEFAULT_SIZE_THRESHOLD] - Threshold separating direct vs link transport + * @param {string} [options.correlation_id=crypto.randomUUID()] - Correlation ID for tracing + * @param {string} [options.msg_purpose="chat"] - Purpose of the message + * @param {string} [options.sender_name="NATSBridge"] - Name of the sender + * @param {string} [options.receiver_name=""] - Name of the receiver (empty means broadcast) + * @param {string} [options.receiver_id=""] - UUID of the receiver (empty means broadcast) + * @param {string} [options.reply_to=""] - Topic to reply to + * @param {string} [options.reply_to_msg_id=""] - Message ID this message is replying to + * @param {boolean} [options.is_publish=true] - Whether to automatically publish the message + * @param {NATSClient|NATS.Connection} [options.nats_connection=null] - Pre-existing NATS connection + * @param {string} [options.msg_id=crypto.randomUUID()] - Message ID + * @param {string} [options.sender_id=crypto.randomUUID()] - Sender ID + * @returns {Promise<[Object, string]>} Tuple of [env, env_json_str] + * + * @example + * // Send a single payload + * const [env, envJsonStr] = await smartsend( + * "/test", + * [["dataname1", data1, "dictionary"]], + * { broker_url: "nats://localhost:4222" } + * ); + * + * // Send multiple payloads + * const [env, envJsonStr] = await smartsend( + * "/test", + * [ + * ["dataname1", data1, "dictionary"], + * ["dataname2", data2, "arrowtable"] + * ], + * { broker_url: "nats://localhost:4222" } + * ); + * + * // Send with pre-existing connection + * const client = await NATSBridge.NATSClient.connect("nats://localhost:4222"); + * const [env, envJsonStr] = await smartsend( + * "/test", + * [["data", myData, "text"]], + * { nats_connection: client } + * ); + */ +async function smartsend(subject, data, options = {}) { + const { + broker_url = DEFAULT_BROKER_URL, + fileserver_url = DEFAULT_FILESERVER_URL, + fileserver_upload_handler = plikOneshotUpload, + size_threshold = DEFAULT_SIZE_THRESHOLD, + correlation_id = crypto.randomUUID(), + msg_purpose = 'chat', + sender_name = 'NATSBridge', + receiver_name = '', + receiver_id = '', + reply_to = '', + reply_to_msg_id = '', + is_publish = true, + nats_connection = null, + msg_id = crypto.randomUUID(), + sender_id = crypto.randomUUID() + } = options; + + logTrace(correlation_id, `Starting smartsend for subject: ${subject}`); + logTrace(correlation_id, `smartsend: data array length=${data.length}`); + + // Debug: Log input data structure + for (let i = 0; i < data.length; i++) { + const [dataname, payloadData, payloadType] = data[i]; + logTrace(correlation_id, `smartsend: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + } + + // Process payloads + const payloads = []; + for (const [dataname, payloadData, payloadType] of data) { + logTrace(correlation_id, `smartsend: Processing payload '${dataname}' type=${payloadType}`); + logTrace(correlation_id, `smartsend: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + + const payloadBytes = await serializeData(payloadData, payloadType); + const payloadSize = payloadBytes.byteLength; + + logTrace(correlation_id, `Serialized payload '${dataname}' (type: ${payloadType}) size: ${payloadSize} bytes`); + + // Debug: Show first 20 bytes of serialized data for table type + if (payloadType === 'table') { + const hexPreview = payloadBytes.slice(0, 20).toString('hex'); + logTrace(correlation_id, `Serialized table data first 20 bytes (hex): ${hexPreview}`); + } + + if (payloadSize < size_threshold) { + // Direct path + const payloadB64 = bufferToBase64(payloadBytes); + logTrace(correlation_id, `Using direct transport for ${payloadSize} bytes, base64 length=${payloadB64.length}`); + + const payload = buildPayload(dataname, payloadType, payloadBytes, 'direct', payloadB64); + payloads.push(payload); + } else { + // Link path + logTrace(correlation_id, `Using link transport, uploading to fileserver`); + + const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes); + + if (response.status !== 200) { + throw new Error(`Failed to upload data to fileserver: ${response.status}`); + } + + logTrace(correlation_id, `Uploaded to URL: ${response.url}`); + + const payload = buildPayload(dataname, payloadType, payloadBytes, 'link', response.url); + payloads.push(payload); + } + } + + // Build envelope + const env = buildEnvelope(subject, payloads, { + correlation_id, + msg_id, + msg_purpose, + sender_name, + sender_id, + receiver_name, + receiver_id, + reply_to, + reply_to_msg_id, + broker_url + }); + + const env_json_str = JSON.stringify(env); + + if (is_publish) { + if (nats_connection) { + await publishMessage(nats_connection, subject, env_json_str, correlation_id); + } else { + await publishMessage(broker_url, subject, env_json_str, correlation_id); + } + } + + return [env, env_json_str]; +} + +/** + * Receive and process NATS message + * + * This function processes incoming NATS messages, handling both direct transport + * (base64 decoded payloads) and link transport (URL-based payloads). + * It deserializes the data based on the transport type and returns the result. + * + * @param {Object} msg - NATS message object with payload property + * @param {Object} options - Optional configuration + * @param {Function} [options.fileserver_download_handler=fetchWithBackoff] - Function to handle fileserver downloads + * @param {number} [options.max_retries=5] - Maximum retry attempts for fetching URL + * @param {number} [options.base_delay=100] - Initial delay for exponential backoff in ms + * @param {number} [options.max_delay=5000] - Maximum delay for exponential backoff in ms + * @returns {Promise} Envelope object with processed payloads + * + * @example + * // Receive and process message + * const env = await smartreceive(msg, { + * fileserver_download_handler: fetchWithBackoff, + * max_retries: 5, + * base_delay: 100, + * max_delay: 5000 + * }); + * // env.payloads is an Array of [dataname, data, type] arrays + * for (const [dataname, data, type] of env.payloads) { + * console.log(`${dataname}: ${data} (type: ${type})`); + * } + */ +async function smartreceive(msg, options = {}) { + const { + fileserver_download_handler = fetchWithBackoff, + max_retries = 5, + base_delay = 100, + max_delay = 5000 + } = options; + + // Debug: Log message object structure + logTrace('smartreceive', `smartreceive: msg object keys: ${Object.keys(msg).join(', ')}`); + logTrace('smartreceive', `smartreceive: msg.data type: ${typeof msg.data}, constructor: ${msg.data?.constructor?.name}`); + logTrace('smartreceive', `smartreceive: msg.payload type: ${typeof msg.payload}, constructor: ${msg.payload?.constructor?.name}`); + + // Parse the JSON envelope + // NATS.js v2.x uses msg.data instead of msg.payload + let payload; + if (msg.data !== undefined) { + payload = typeof msg.data === 'string' ? msg.data : Buffer.from(msg.data).toString('utf8'); + } else if (msg.payload !== undefined) { + payload = typeof msg.payload === 'string' ? msg.payload : Buffer.from(msg.payload).toString('utf8'); + } else { + throw new Error('Message has neither data nor payload property'); + } + + logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); + + // Debug: Show first 200 chars of payload + const payloadPreview = payload.substring(0, 200); + logTrace('smartreceive', `smartreceive: payload preview: ${payloadPreview}`); + + let envJsonObj; + try { + envJsonObj = JSON.parse(payload); + } catch (e) { + logTrace('smartreceive', `smartreceive: JSON parse failed: ${e.message}`); + throw e; + } + + logTrace(envJsonObj.correlation_id, 'Processing received message'); + logTrace(envJsonObj.correlation_id, `smartreceive: envelope has ${envJsonObj.payloads.length} payloads`); + + // Process all payloads in the envelope + const payloadsList = []; + const numPayloads = envJsonObj.payloads.length; + + logTrace(envJsonObj.correlation_id, `smartreceive: Processing ${numPayloads} payloads`); + + for (let i = 0; i < numPayloads; i++) { + const payloadObj = envJsonObj.payloads[i]; + const transport = payloadObj.transport; + const dataname = payloadObj.dataname; + const payloadType = payloadObj.payload_type; + + logTrace(envJsonObj.correlation_id, `smartreceive: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); + + if (transport === 'direct') { + logTrace(envJsonObj.correlation_id, `Direct transport - decoding payload '${dataname}'`); + + // Extract base64 payload from the payload + const payloadB64 = payloadObj.data; + logTrace(envJsonObj.correlation_id, `Direct transport: base64 length=${payloadB64?.length}`); + + // Decode Base64 payload + const payloadBytes = Buffer.from(payloadB64, 'base64'); + logTrace(envJsonObj.correlation_id, `Direct transport: decoded bytes=${payloadBytes.length}`); + + // Deserialize based on type + const dataType = payloadObj.payload_type; + const data = await deserializeData(payloadBytes, dataType, envJsonObj.correlation_id); + logTrace(envJsonObj.correlation_id, `Direct transport: deserialized data type=${typeof data}, constructor=${data?.constructor?.name}`); + + payloadsList.push([dataname, data, dataType]); + } else if (transport === 'link') { + // Extract download URL from the payload + const url = payloadObj.data; + logTrace(envJsonObj.correlation_id, `Link transport - fetching '${dataname}' from URL: ${url}`); + + // Fetch with exponential backoff using the download handler + const downloadedData = await fileserver_download_handler( + url, + max_retries, + base_delay, + max_delay, + envJsonObj.correlation_id + ); + + // Deserialize based on type + const dataType = payloadObj.payload_type; + const data = await deserializeData(downloadedData, dataType, envJsonObj.correlation_id); + + payloadsList.push([dataname, data, dataType]); + } else { + throw new Error(`Unknown transport type for payload '${dataname}': ${transport}`); + } + } + + logTrace(envJsonObj.correlation_id, `smartreceive: Successfully processed all ${payloadsList.length} payloads`); + envJsonObj.payloads = payloadsList; + return envJsonObj; +} + +// ---------------------------------------------- Module Exports ---------------------------------------------- // + +const NATSBridge = { + /** + * NATS client class for connection management + */ + NATSClient, + + /** + * Send data via NATS with automatic transport selection + */ + smartsend, + + /** + * Receive and process NATS message + */ + smartreceive, + + /** + * Upload data to plik server in one-shot mode + */ + plikOneshotUpload, + + /** + * Fetch data from URL with exponential backoff + */ + fetchWithBackoff, + + /** + * Default constants + */ + DEFAULT_SIZE_THRESHOLD, + DEFAULT_BROKER_URL, + DEFAULT_FILESERVER_URL +}; + +module.exports = NATSBridge; \ No newline at end of file diff --git a/src/natsbridge.py b/src/natsbridge.py new file mode 100644 index 0000000..98e456a --- /dev/null +++ b/src/natsbridge.py @@ -0,0 +1,843 @@ +""" +NATSBridge - Cross-Platform Bi-Directional Data Bridge +Python Desktop Implementation + +This module provides functionality for sending and receiving data across network boundaries +using NATS as the message bus, with support for both direct payload transport and +URL-based transport for larger payloads. + +@package natsbridge +""" + +import asyncio +import base64 +import json +import uuid +from datetime import datetime +from typing import Any, Callable, Dict, List, Tuple, Union +import aiohttp + +try: + import pyarrow as arrow + import pyarrow.ipc as ipc + ARROW_AVAILABLE = True +except ImportError: + ARROW_AVAILABLE = False + +try: + import nats + from nats.aio.client import Client as NATSClient + NATS_AVAILABLE = True +except ImportError: + NATS_AVAILABLE = False + +# ---------------------------------------------- Constants ---------------------------------------------- # + +""" +Default size threshold for switching from direct to link transport (1MB) +""" +DEFAULT_SIZE_THRESHOLD = 1_000_000 + +""" +Default NATS server URL +""" +DEFAULT_BROKER_URL = "nats://localhost:4222" + +""" +Default HTTP file server URL for link transport +""" +DEFAULT_FILESERVER_URL = "http://localhost:8080" + + +# ---------------------------------------------- Utility Functions ---------------------------------------------- # + +def log_trace(correlation_id: str, message: str) -> None: + """ + Log a trace message with correlation ID and timestamp. + + Args: + correlation_id: Correlation ID for tracing + message: Message content to log + """ + timestamp = datetime.utcnow().isoformat() + 'Z' + print(f"[{timestamp}] [Correlation: {correlation_id}] {message}") + + +# ---------------------------------------------- Serialization Functions ---------------------------------------------- # + +def _serialize_data(data: Any, payload_type: str) -> bytes: + """ + Serialize data according to specified format. + + Args: + data: Data to serialize (string for "text", JSON-serializable for "dictionary", + table-like for "arrowtable"/"jsontable", binary for "image", "audio", "video", "binary") + payload_type: Target format: "text", "dictionary", "arrowtable", "jsontable", + "image", "audio", "video", "binary" + + Returns: + Binary representation of the serialized data + + Raises: + Error: If payload_type is not one of the supported types + Error: If payload_type is "image", "audio", or "video" but data is not bytes + Error: If payload_type is "arrowtable" but data is not a pandas DataFrame or pyarrow Table + Error: If payload_type is "jsontable" but data is not a list of dicts + """ + if payload_type == 'text': + if isinstance(data, str): + return data.encode('utf-8') + else: + raise ValueError('Text data must be a string') + elif payload_type == 'dictionary': + json_str = json.dumps(data) + return json_str.encode('utf-8') + elif payload_type == 'arrowtable': + if not ARROW_AVAILABLE: + raise RuntimeError('pyarrow not available for arrowtable serialization') + + import io + buf = io.BytesIO() + + import pandas as pd + if isinstance(data, pd.DataFrame): + table = arrow.Table.from_pandas(data) + sink = ipc.new_file(buf, table.schema) + ipc.write_table(table, sink) + sink.close() + return buf.getvalue() + elif isinstance(data, arrow.Table): + sink = ipc.new_file(buf, data.schema) + ipc.write_table(data, sink) + sink.close() + return buf.getvalue() + else: + raise ValueError('Arrow table data must be a pandas DataFrame or pyarrow Table') + elif payload_type == 'jsontable': + # Serialize list of dicts to JSON format + if isinstance(data, list) and all(isinstance(row, dict) for row in data): + json_str = json.dumps(data) + return json_str.encode('utf-8') + else: + raise ValueError('JSON table data must be a list of dicts') + elif payload_type == 'image': + if isinstance(data, (bytes, bytearray)): + return bytes(data) + else: + raise ValueError('Image data must be bytes') + elif payload_type == 'audio': + if isinstance(data, (bytes, bytearray)): + return bytes(data) + else: + raise ValueError('Audio data must be bytes') + elif payload_type == 'video': + if isinstance(data, (bytes, bytearray)): + return bytes(data) + else: + raise ValueError('Video data must be bytes') + elif payload_type == 'binary': + if isinstance(data, (bytes, bytearray)): + return bytes(data) + else: + raise ValueError('Binary data must be bytes') + else: + raise ValueError(f'Unknown payload_type: {payload_type}') + + +def _deserialize_data(data: bytes, payload_type: str, correlation_id: str) -> Any: + """ + Deserialize bytes to data based on type. + + Args: + data: Serialized data as bytes + payload_type: Data type ("text", "dictionary", "arrowtable", "jsontable", + "image", "audio", "video", "binary") + correlation_id: Correlation ID for logging + + Returns: + Deserialized data (String for "text", DataFrame for "arrowtable", + Vector{Dict} for "jsontable"/"dictionary", bytes for "image", "audio", "video", "binary") + + Raises: + Error: If payload_type is not one of the supported types + """ + if payload_type == 'text': + return data.decode('utf-8') + elif payload_type == 'dictionary': + json_str = data.decode('utf-8') + return json.loads(json_str) + elif payload_type == 'arrowtable': + if not ARROW_AVAILABLE: + raise RuntimeError('pyarrow not available for arrowtable deserialization') + + import io + buf = io.BytesIO(data) + reader = ipc.open_file(buf) + return reader.read_all().to_pandas() + elif payload_type == 'jsontable': + # Deserialize JSON to list of dicts + json_str = data.decode('utf-8') + return json.loads(json_str) + elif payload_type == 'image': + return data + elif payload_type == 'audio': + return data + elif payload_type == 'video': + return data + elif payload_type == 'binary': + return data + else: + raise ValueError(f'Unknown payload_type: {payload_type}') + + +# ---------------------------------------------- File Server Handlers ---------------------------------------------- # + +async def plik_oneshot_upload( + file_server_url: str, + dataname: str, + data: bytes +) -> Dict[str, Any]: + """ + Upload data to plik server in one-shot mode. + + This function uploads a raw byte array to a plik server in one-shot mode (no upload session). + It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, + retrieves an upload ID and token, then uploads the file data as multipart form data using the token. + + Args: + file_server_url: Base URL of the plik server (e.g., "http://localhost:8080") + dataname: Name of the file being uploaded + data: Raw byte data of the file content + + Returns: + Dict with keys: + - "status": HTTP server response status + - "uploadid": ID of the one-shot upload session + - "fileid": ID of the uploaded file within the session + - "url": Full URL to download the uploaded file + + Example: + >>> fileserver_url = "http://localhost:8080" + >>> dataname = "test.txt" + >>> data = b"hello world" + >>> result = await plik_oneshot_upload(file_server_url, dataname, data) + >>> result["status"], result["uploadid"], result["fileid"], result["url"] + """ + async with aiohttp.ClientSession() as session: + # Get upload id + url_getUploadID = f"{file_server_url}/upload" + headers = {'Content-Type': 'application/json'} + body = json.dumps({"OneShot": True}) + + async with session.post(url_getUploadID, headers=headers, data=body) as response: + response_json = await response.json() + uploadid = response_json['id'] + uploadtoken = response_json['uploadToken'] + + # Upload file + url_upload = f"{file_server_url}/file/{uploadid}" + headers = {'X-UploadToken': uploadtoken} + + form = aiohttp.FormData() + form.add_field('file', data, filename=dataname, content_type='application/octet-stream') + + async with session.post(url_upload, headers=headers, data=form) as upload_response: + upload_json = await upload_response.json() + fileid = upload_json['id'] + + url = f"{file_server_url}/file/{uploadid}/{fileid}/{dataname}" + + return { + 'status': upload_response.status, + 'uploadid': uploadid, + 'fileid': fileid, + 'url': url + } + + +async def fetch_with_backoff( + url: str, + max_retries: int, + base_delay: int, + max_delay: int, + correlation_id: str +) -> bytes: + """ + Fetch data from URL with exponential backoff. + + This internal function retrieves data from a URL with retry logic using + exponential backoff to handle transient failures. + + Args: + url: URL to fetch from + max_retries: Maximum number of retry attempts + base_delay: Initial delay in milliseconds + max_delay: Maximum delay in milliseconds + correlation_id: Correlation ID for logging + + Returns: + Fetched data as bytes + + Raises: + Error: If all retry attempts fail + + Example: + >>> data = await fetch_with_backoff("http://example.com/file.zip", 5, 100, 5000, "correlation123") + """ + delay = base_delay + + for attempt in range(1, max_retries + 1): + try: + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + if response.status == 200: + log_trace(correlation_id, f"Successfully fetched data from {url} on attempt {attempt}") + return await response.read() + else: + raise Exception(f"Failed to fetch: {response.status}") + except Exception as e: + log_trace(correlation_id, f"Attempt {attempt} failed: {type(e).__name__}") + + if attempt < max_retries: + await asyncio.sleep(delay / 1000.0) + delay = min(delay * 2, max_delay) + + raise Exception(f"Failed to fetch data after {max_retries} attempts") + + +# ---------------------------------------------- NATS Client ---------------------------------------------- # + +class NATSClient: + """NATS client wrapper for connection management.""" + + def __init__(self, url: str = DEFAULT_BROKER_URL): + """ + Create a new NATS client. + + Args: + url: NATS server URL + """ + self.url = url + self._client: NATSClient = None + + async def connect(self) -> NATSClient: + """ + Connect to NATS server. + + Returns: + NATS client instance + """ + if NATS_AVAILABLE: + self._client = nats.connect(self.url) + await self._client + else: + raise RuntimeError('nats-py not available') + return self._client + + async def publish(self, subject: str, message: str, correlation_id: str = "") -> None: + """ + Publish message to NATS subject. + + Args: + subject: NATS subject to publish to + message: Message to publish + correlation_id: Correlation ID for logging + """ + if self._client: + await self._client.publish(subject, message) + if correlation_id: + log_trace(correlation_id, f"Message published to {subject}") + + async def close(self) -> None: + """Close the NATS connection.""" + if self._client: + await self._client.drain() + await self._client.close() + + +# ---------------------------------------------- Core Functions ---------------------------------------------- # + +def _build_envelope( + subject: str, + payloads: List[Dict[str, Any]], + options: Dict[str, Any] +) -> Dict[str, Any]: + """ + Build message envelope from payloads and metadata. + + Args: + subject: NATS subject + payloads: Array of payload objects + options: Envelope metadata options + + Returns: + Envelope object + """ + return { + 'correlation_id': options['correlation_id'], + 'msg_id': options['msg_id'], + 'timestamp': datetime.utcnow().isoformat() + 'Z', + 'send_to': subject, + 'msg_purpose': options['msg_purpose'], + 'sender_name': options['sender_name'], + 'sender_id': options['sender_id'], + 'receiver_name': options['receiver_name'], + 'receiver_id': options['receiver_id'], + 'reply_to': options['reply_to'], + 'reply_to_msg_id': options['reply_to_msg_id'], + 'broker_url': options['broker_url'], + 'metadata': options.get('metadata', {}), + 'payloads': payloads + } + + +def _build_payload( + dataname: str, + payload_type: str, + payload_bytes: bytes, + transport: str, + data: Union[str, bytes] +) -> Dict[str, Any]: + """ + Build payload object from serialized data. + + Args: + dataname: Name of the payload + payload_type: Type of the payload + payload_bytes: Serialized payload bytes + transport: Transport type ("direct" or "link") + data: Data (base64 for direct, URL for link) + + Returns: + Payload object + """ + # Determine encoding based on payload type (matching Julia/JS implementation) + encoding = 'base64' + if payload_type == 'jsontable': + encoding = 'json' + elif payload_type == 'arrowtable': + encoding = 'arrow-ipc' + + return { + 'id': str(uuid.uuid4()), + 'dataname': dataname, + 'payload_type': payload_type, + 'transport': transport, + 'encoding': encoding, + 'size': len(payload_bytes), + 'data': data, + 'metadata': {'payload_bytes': len(payload_bytes)} if transport == 'direct' else {} + } + + +async def publish_message( + broker_url_or_client: Union[str, NATSClient, Any], + subject: str, + message: str, + correlation_id: str +) -> None: + """ + Publish message to NATS. + + Args: + broker_url_or_client: NATS URL, client, or connection + subject: NATS subject to publish to + message: JSON message to publish + correlation_id: Correlation ID for tracing + """ + if isinstance(broker_url_or_client, NATSClient): + client = broker_url_or_client + elif NATS_AVAILABLE and hasattr(broker_url_or_client, 'publish'): + # Direct NATS client connection + await broker_url_or_client.publish(subject, message) + log_trace(correlation_id, f"Message published to {subject}") + return + else: + # String URL - create new client + client = NATSClient(broker_url_or_client) + await client.connect() + + await client.publish(subject, message, correlation_id) + + if isinstance(broker_url_or_client, NATSClient): + await broker_url_or_client.close() + elif not (NATS_AVAILABLE and hasattr(broker_url_or_client, 'publish')): + await client.close() + + +async def smartsend( + subject: str, + data: List[Tuple[str, Any, str]], + broker_url: str = DEFAULT_BROKER_URL, + fileserver_url: str = DEFAULT_FILESERVER_URL, + fileserver_upload_handler: Callable = plik_oneshot_upload, + size_threshold: int = DEFAULT_SIZE_THRESHOLD, + correlation_id: str = None, + msg_purpose: str = "chat", + sender_name: str = "NATSBridge", + receiver_name: str = "", + receiver_id: str = "", + reply_to: str = "", + reply_to_msg_id: str = "", + is_publish: bool = True, + nats_connection: Any = None, + msg_id: str = None, + sender_id: str = None +) -> Tuple[Dict, str]: + """ + Send data via NATS with automatic transport selection. + + This function intelligently routes data delivery based on payload size. + If the serialized payload is smaller than size_threshold, it encodes the data as Base64 + and publishes directly over NATS. Otherwise, it uploads the data to a fileserver + and publishes only the download URL over NATS. + + Args: + subject: NATS subject to publish the message to + data: List of (dataname, data, type) tuples to send + - dataname: Name of the payload + - data: The actual data to send + - type: Payload type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" + broker_url: URL of the NATS server + fileserver_url: URL of the HTTP file server for large payloads + fileserver_upload_handler: Function to handle fileserver uploads (must return Dict with "status", + "uploadid", "fileid", "url" keys) + size_threshold: Threshold in bytes separating direct vs link transport + correlation_id: Correlation ID for tracing (auto-generated UUID if not provided) + msg_purpose: Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. + sender_name: Name of the sender + receiver_name: Name of the receiver (empty string means broadcast) + receiver_id: UUID of the receiver (empty string means broadcast) + reply_to: Topic to reply to (empty string if no reply expected) + reply_to_msg_id: Message ID this message is replying to + is_publish: Whether to automatically publish the message to NATS + nats_connection: Pre-existing NATS connection (if provided, uses this connection instead of + creating a new one; saves connection establishment overhead) + msg_id: Message ID (auto-generated UUID if not provided) + sender_id: Sender ID (auto-generated UUID if not provided) + + Returns: + Tuple of (env, env_json_str) where: + - env: Dict containing all metadata and payloads + - env_json_str: JSON string for publishing to NATS + + Example: + >>> # Send a single payload (still wrapped in a list) + >>> data = {"key": "value"} + >>> env, env_json_str = await smartsend( + ... "my.subject", + ... [("dataname1", data, "dictionary")], + ... broker_url="nats://localhost:4222" + ... ) + >>> + >>> # Send multiple payloads with different types + >>> data1 = {"key1": "value1"} + >>> data2 = [1, 2, 3, 4, 5] + >>> env, env_json_str = await smartsend( + ... "my.subject", + ... [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")] + ... ) + >>> + >>> # Send a large array using fileserver upload + >>> data = list(range(10_000_000)) # ~80 MB + >>> env, env_json_str = await smartsend( + ... "large.data", + ... [("large_table", data, "arrowtable")] + ... ) + >>> + >>> # Send jsontable (JSON format for human-readable tabular data) + >>> users = [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] + >>> env, env_json_str = await smartsend( + ... "json.data", + ... [("users", users, "jsontable")] + ... ) + >>> + >>> # Mixed content (e.g., chat with text and image) + >>> env, env_json_str = await smartsend( + ... "chat.subject", + ... [ + ... ("message_text", "Hello!", "text"), + ... ("user_image", image_data, "image"), + ... ("audio_clip", audio_data, "audio") + ... ] + ... ) + >>> + >>> # Publish the JSON string directly using NATS request-reply pattern + >>> # reply = await nats.request(broker_url, subject, env_json_str, reply_to=reply_to_topic) + """ + if correlation_id is None: + correlation_id = str(uuid.uuid4()) + if msg_id is None: + msg_id = str(uuid.uuid4()) + if sender_id is None: + sender_id = str(uuid.uuid4()) + + log_trace(correlation_id, f"Starting smartsend for subject: {subject}") + + # Process payloads + payloads = [] + for dataname, payload_data, payload_type in data: + payload_bytes = _serialize_data(payload_data, payload_type) + payload_size = len(payload_bytes) + + log_trace(correlation_id, f"Serialized payload '{dataname}' (type: {payload_type}) size: {payload_size} bytes") + + if payload_size < size_threshold: + # Direct path + payload_b64 = base64.b64encode(payload_bytes).decode('utf-8') + log_trace(correlation_id, f"Using direct transport for {payload_size} bytes") + + payload = _build_payload(dataname, payload_type, payload_bytes, 'direct', payload_b64) + payloads.append(payload) + else: + # Link path + log_trace(correlation_id, "Using link transport, uploading to fileserver") + + response = await fileserver_upload_handler(fileserver_url, dataname, payload_bytes) + + if response['status'] != 200: + raise Exception(f"Failed to upload data to fileserver: {response['status']}") + + log_trace(correlation_id, f"Uploaded to URL: {response['url']}") + + payload = _build_payload(dataname, payload_type, payload_bytes, 'link', response['url']) + payloads.append(payload) + + # Build envelope + env = _build_envelope(subject, payloads, { + 'correlation_id': correlation_id, + 'msg_id': msg_id, + 'msg_purpose': msg_purpose, + 'sender_name': sender_name, + 'sender_id': sender_id, + 'receiver_name': receiver_name, + 'receiver_id': receiver_id, + 'reply_to': reply_to, + 'reply_to_msg_id': reply_to_msg_id, + 'broker_url': broker_url + }) + + env_json_str = json.dumps(env) + + if is_publish: + if nats_connection: + await publish_message(nats_connection, subject, env_json_str, correlation_id) + else: + await publish_message(broker_url, subject, env_json_str, correlation_id) + + return env, env_json_str + + +async def smartreceive( + msg: Any, + fileserver_download_handler: Callable = fetch_with_backoff, + max_retries: int = 5, + base_delay: int = 100, + max_delay: int = 5000 +) -> Dict[str, Any]: + """ + Receive and process NATS messages. + + This function processes incoming NATS messages, handling both direct transport + (base64 decoded payloads) and link transport (URL-based payloads). + It deserializes the data based on the transport type and returns the result. + + Args: + msg: NATS message to process + fileserver_download_handler: Function to handle downloading data from file server URLs + max_retries: Maximum retry attempts for fetching URL + base_delay: Initial delay for exponential backoff in ms + max_delay: Maximum delay for exponential backoff in ms + + Returns: + Dict with envelope metadata and payloads field containing List[Tuple[str, Any, str]] + + Example: + >>> # Receive and process message + >>> env = await smartreceive(msg, fileserver_download_handler=fetch_with_backoff) + >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] + >>> # Access payloads: for dataname, data, type_ in env["payloads"] + >>> for dataname, data, type_ in env["payloads"]: + >>> print(f"{dataname}: {data} (type: {type_})") + """ + # Parse the JSON envelope + if isinstance(msg, dict): + # Already parsed + env_json_obj = msg + elif hasattr(msg, 'payload'): + # NATS message object + payload = msg.payload if isinstance(msg.payload, str) else msg.payload.decode('utf-8') + env_json_obj = json.loads(payload) + else: + # Assume it's already a JSON string or dict + env_json_obj = json.loads(msg) if isinstance(msg, str) else msg + + log_trace(env_json_obj['correlation_id'], "Processing received message") + + # Process all payloads in the envelope + payloads_list = [] + num_payloads = len(env_json_obj['payloads']) + + for i in range(num_payloads): + payload_obj = env_json_obj['payloads'][i] + transport = payload_obj['transport'] + dataname = payload_obj['dataname'] + + if transport == 'direct': + log_trace(env_json_obj['correlation_id'], f"Direct transport - decoding payload '{dataname}'") + + # Extract base64 payload from the payload + payload_b64 = payload_obj['data'] + + # Decode Base64 payload + payload_bytes = base64.b64decode(payload_b64) + + # Deserialize based on type + data_type = payload_obj['payload_type'] + data = _deserialize_data(payload_bytes, data_type, env_json_obj['correlation_id']) + + payloads_list.append((dataname, data, data_type)) + elif transport == 'link': + # Extract download URL from the payload + url = payload_obj['data'] + log_trace(env_json_obj['correlation_id'], f"Link transport - fetching '{dataname}' from URL: {url}") + + # Fetch with exponential backoff using the download handler + downloaded_data = await fileserver_download_handler( + url, + max_retries, + base_delay, + max_delay, + env_json_obj['correlation_id'] + ) + + # Deserialize based on type + data_type = payload_obj['payload_type'] + data = _deserialize_data(downloaded_data, data_type, env_json_obj['correlation_id']) + + payloads_list.append((dataname, data, data_type)) + else: + raise Exception(f"Unknown transport type for payload '{dataname}': {transport}") + + env_json_obj['payloads'] = payloads_list + return env_json_obj + + +# ---------------------------------------------- Module Exports ---------------------------------------------- # + +class NATSBridge: + """ + Cross-platform NATS bridge implementation. + + This class provides a convenient interface for NATSBridge functionality, + encapsulating the main functions and providing a class-based API. + """ + + DEFAULT_SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD + DEFAULT_BROKER_URL = DEFAULT_BROKER_URL + DEFAULT_FILESERVER_URL = DEFAULT_FILESERVER_URL + + def __init__(self, broker_url: str = None, fileserver_url: str = None): + """ + Initialize NATSBridge. + + Args: + broker_url: NATS server URL (defaults to DEFAULT_BROKER_URL) + fileserver_url: HTTP file server URL (defaults to DEFAULT_FILESERVER_URL) + """ + self.broker_url = broker_url or self.DEFAULT_BROKER_URL + self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL + + async def smartsend( + self, + subject: str, + data: List[Tuple[str, Any, str]], + **kwargs + ) -> Tuple[Dict, str]: + """ + Send data via NATS. + + Args: + subject: NATS subject to publish to + data: List of (dataname, data, type) tuples + **kwargs: Additional options passed to smartsend + + Returns: + Tuple of (env, env_json_str) + """ + kwargs['broker_url'] = kwargs.get('broker_url', self.broker_url) + kwargs['fileserver_url'] = kwargs.get('fileserver_url', self.fileserver_url) + return await smartsend(subject, data, **kwargs) + + async def smartreceive( + self, + msg: Any, + **kwargs + ) -> Dict[str, Any]: + """ + Receive and process NATS message. + + Args: + msg: NATS message to process + **kwargs: Additional options passed to smartreceive + + Returns: + Dict with envelope metadata and payloads + """ + return await smartreceive(msg, **kwargs) + + +# Convenience functions for module-level usage +def send( + subject: str, + data: List[Tuple[str, Any, str]], + **kwargs +) -> Tuple[Dict, str]: + """ + Convenience function for sending data. + + Args: + subject: NATS subject to publish to + data: List of (dataname, data, type) tuples + **kwargs: Additional options + + Returns: + Tuple of (env, env_json_str) + """ + return asyncio.run(smartsend(subject, data, **kwargs)) + + +def receive( + msg: Any, + **kwargs +) -> Dict[str, Any]: + """ + Convenience function for receiving messages. + + Args: + msg: NATS message to process + **kwargs: Additional options + + Returns: + Dict with envelope metadata and payloads + """ + return asyncio.run(smartreceive(msg, **kwargs)) + + +__all__ = [ + 'smartsend', + 'smartreceive', + 'plik_oneshot_upload', + 'fetch_with_backoff', + 'NATSBridge', + 'send', + 'receive', + 'DEFAULT_SIZE_THRESHOLD', + 'DEFAULT_BROKER_URL', + 'DEFAULT_FILESERVER_URL', + 'NATSClient', + '_serialize_data', + '_deserialize_data', + 'log_trace', + 'publish_message' +] \ No newline at end of file diff --git a/src/natsbridge_mpy.py b/src/natsbridge_mpy.py new file mode 100644 index 0000000..d99631f --- /dev/null +++ b/src/natsbridge_mpy.py @@ -0,0 +1,673 @@ +""" +NATSBridge - Cross-Platform Bi-Directional Data Bridge +MicroPython Implementation + +This module provides functionality for sending and receiving data across network boundaries +using NATS as the message bus, with support for both direct payload transport and +URL-based transport for larger payloads. + +Note: MicroPython has significant constraints compared to desktop implementations: +- Limited memory (~256KB - 1MB) +- No Arrow IPC support (memory constraints) +- Synchronous API (no async/await) +- Lower size threshold for direct transport +""" + +import network +import time +import json +import base64 +import uos +import struct +import random + +# ---------------------------------------------- Constants ---------------------------------------------- # + +""" +Default size threshold for switching from direct to link transport (100KB for MicroPython) +""" +DEFAULT_SIZE_THRESHOLD = 100000 + +""" +Default NATS server URL +""" +DEFAULT_BROKER_URL = "nats://localhost:4222" + +""" +Default HTTP file server URL for link transport +""" +DEFAULT_FILESERVER_URL = "http://localhost:8080" + +""" +Hard limit for payload size in MicroPython (50KB) +""" +MAX_PAYLOAD_SIZE = 50000 + + +# ---------------------------------------------- Utility Functions ---------------------------------------------- # + +def log_trace(correlation_id, message): + """ + Log a trace message with correlation ID and timestamp. + + Args: + correlation_id: Correlation ID for tracing + message: Message content to log + """ + timestamp = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime()) + print(f"[{timestamp}] [Correlation: {correlation_id}] {message}") + + +def _generate_uuid(): + """ + Generate a simple UUID compatible with MicroPython. + + Returns: + UUID string + """ + # Generate a simple UUID-like string + # Format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + hex_chars = '0123456789abcdef' + uuid_str = ''.join([random.choice(hex_chars) for _ in range(32)]) + # Insert hyphens at proper positions + return f"{uuid_str[:8]}-{uuid_str[8:12]}-{uuid_str[12:16]}-{uuid_str[16:20]}-{uuid_str[20:]}" + + +# ---------------------------------------------- Serialization Functions ---------------------------------------------- # + +def _serialize_data(data, payload_type): + """ + Serialize data according to specified format. + + Args: + data: Data to serialize (string for "text", dict for "dictionary", + bytes for "image", "audio", "video", "binary") + payload_type: Target format: "text", "dictionary", "image", "audio", "video", "binary" + + Returns: + Binary representation of the serialized data + + Note: + MicroPython does not support "table" type due to memory constraints. + + Raises: + ValueError: If payload_type is not one of the supported types + """ + if payload_type == 'text': + if isinstance(data, str): + return data.encode('utf-8') + else: + raise ValueError('Text data must be a string') + elif payload_type == 'dictionary': + json_str = json.dumps(data) + return json_str.encode('utf-8') + elif payload_type in ('image', 'audio', 'video', 'binary'): + if isinstance(data, (bytes, bytearray, memoryview)): + return bytes(data) + else: + raise ValueError(f'{payload_type} data must be bytes') + else: + raise ValueError(f'Unknown payload_type: {payload_type}') + + +def _deserialize_data(data, payload_type): + """ + Deserialize bytes to data based on type. + + Args: + data: Serialized data as bytes + payload_type: Data type ("text", "dictionary", "image", "audio", "video", "binary") + + Returns: + Deserialized data (String for "text", dict for "dictionary", bytes for others) + + Note: + MicroPython does not support "table" type due to memory constraints. + + Raises: + ValueError: If payload_type is not one of the supported types + """ + if payload_type == 'text': + return data.decode('utf-8') + elif payload_type == 'dictionary': + json_str = data.decode('utf-8') + return json.loads(json_str) + elif payload_type in ('image', 'audio', 'video', 'binary'): + return data + else: + raise ValueError(f'Unknown payload_type: {payload_type}') + + +# ---------------------------------------------- File Server Handlers ---------------------------------------------- # + +def _sync_fileserver_upload(file_server_url, dataname, data): + """ + Synchronous file upload to HTTP server. + + Note: + This is a simplified implementation for MicroPython. + In practice, would use network.HTTP or similar. + Currently raises NotImplementedError as file upload is not fully supported. + + Args: + file_server_url: Base URL of the file server + dataname: Name of the file being uploaded + data: Raw byte data of the file content + + Returns: + Dict with keys: 'status', 'url' + + Raises: + NotImplementedError: File upload is not implemented in MicroPython + """ + raise NotImplementedError("File upload not fully implemented in MicroPython. " + "Use direct transport only for memory-constrained devices.") + + +def _sync_fileserver_download(url, max_retries, base_delay, max_delay, correlation_id): + """ + Synchronous file download with exponential backoff. + + Note: + This is a simplified implementation for MicroPython. + In practice, would use network.HTTP or similar. + Currently raises NotImplementedError as file download is not fully supported. + + Args: + url: URL to download from + max_retries: Maximum retry attempts + base_delay: Initial delay in ms + max_delay: Maximum delay in ms + correlation_id: Correlation ID for logging + + Returns: + Downloaded bytes + + Raises: + NotImplementedError: File download is not implemented in MicroPython + """ + raise NotImplementedError("File download not fully implemented in MicroPython. " + "Use direct transport only for memory-constrained devices.") + + +# ---------------------------------------------- NATS Client ---------------------------------------------- # + +class NATSClient: + """ + NATS client wrapper for MicroPython. + + Note: + This is a simplified implementation for MicroPython. + Full NATS client implementation would require additional network stack support. + """ + + def __init__(self, url=DEFAULT_BROKER_URL): + """ + Initialize NATS client. + + Args: + url: NATS server URL + """ + self.url = url + self._connected = False + + def connect(self): + """ + Connect to NATS server. + + Note: + This is a placeholder implementation. + Actual NATS client would require network stack support. + + Returns: + True if connected, False otherwise + """ + # Placeholder - actual implementation would connect to NATS server + self._connected = True + return self._connected + + def publish(self, subject, message): + """ + Publish message to NATS subject. + + Note: + This is a placeholder implementation. + Actual NATS client would require network stack support. + + Args: + subject: NATS subject to publish to + message: Message to publish + """ + if not self._connected: + raise RuntimeError("Not connected to NATS server") + # Placeholder - actual implementation would publish to NATS + print(f"[NATS] Publish to {subject}: {message[:50]}...") + + def close(self): + """Close the NATS connection.""" + self._connected = False + + +# ---------------------------------------------- Core Functions ---------------------------------------------- # + +def _build_envelope(subject, payloads, options): + """ + Build message envelope from payloads and metadata. + + Args: + subject: NATS subject + payloads: Array of payload objects + options: Envelope metadata options + + Returns: + Envelope dict + """ + return { + 'correlation_id': options['correlation_id'], + 'msg_id': options['msg_id'], + 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime()), + 'send_to': subject, + 'msg_purpose': options['msg_purpose'], + 'sender_name': options['sender_name'], + 'sender_id': options['sender_id'], + 'receiver_name': options['receiver_name'], + 'receiver_id': options['receiver_id'], + 'reply_to': options['reply_to'], + 'reply_to_msg_id': options['reply_to_msg_id'], + 'broker_url': options['broker_url'], + 'metadata': {}, + 'payloads': payloads + } + + +def _build_payload(dataname, payload_type, payload_bytes, transport, data): + """ + Build payload object from serialized data. + + Args: + dataname: Name of the payload + payload_type: Type of the payload + payload_bytes: Serialized payload bytes + transport: Transport type ("direct" or "link") + data: Data (base64 for direct, URL for link) + + Returns: + Payload dict + """ + return { + 'id': _generate_uuid(), + 'dataname': dataname, + 'payload_type': payload_type, + 'transport': transport, + 'encoding': 'base64' if transport == 'direct' else 'none', + 'size': len(payload_bytes), + 'data': data, + 'metadata': {'payload_bytes': len(payload_bytes)} if transport == 'direct' else {} + } + + +def _publish(subject, message, correlation_id): + """ + Publish message to NATS. + + Note: + This is a simplified implementation for MicroPython. + + Args: + subject: NATS subject to publish to + message: JSON message to publish + correlation_id: Correlation ID for logging + """ + log_trace(correlation_id, f"Publishing to {subject}") + # Placeholder - actual implementation would use NATSClient + # client = NATSClient() + # client.connect() + # client.publish(subject, message) + # client.close() + + +def smartsend(subject, data, **kwargs): + """ + Send data via NATS with automatic transport selection. + + This function intelligently routes data delivery based on payload size. + If the serialized payload is smaller than size_threshold, it encodes the data as Base64 + and publishes directly over NATS. Otherwise, it uploads the data to a fileserver + and publishes only the download URL over NATS. + + Note: + MicroPython has memory constraints, so the default size_threshold is lower (100KB). + Table type is not supported due to memory constraints. + + Args: + subject: NATS subject to publish the message to + data: List of (dataname, data, type) tuples to send + - dataname: Name of the payload + - data: The actual data to send + - type: Payload type: "text", "dictionary", "image", "audio", "video", "binary" + broker_url: NATS server URL (default: DEFAULT_BROKER_URL) + fileserver_url: HTTP file server URL (default: DEFAULT_FILESERVER_URL) + fileserver_upload_handler: Function to handle fileserver uploads (default: _sync_fileserver_upload) + size_threshold: Threshold in bytes separating direct vs link transport (default: 100000) + correlation_id: Correlation ID for tracing (auto-generated if not provided) + msg_purpose: Purpose of the message (default: "chat") + sender_name: Name of the sender (default: "NATSBridge") + receiver_name: Name of the receiver (empty means broadcast) + receiver_id: UUID of the receiver (empty means broadcast) + reply_to: Topic to reply to (empty if no reply expected) + reply_to_msg_id: Message ID this message is replying to + is_publish: Whether to automatically publish the message (default: True) + msg_id: Message ID (auto-generated if not provided) + sender_id: Sender ID (auto-generated if not provided) + + Returns: + Tuple of (env, env_json_str) where: + - env: Dict containing all metadata and payloads + - env_json_str: JSON string for publishing to NATS + + Example: + >>> # Send text payload + >>> env, env_json_str = NATSBridge.smartsend( + ... "/chat", + ... [("message", "Hello!", "text")], + ... broker_url="nats://localhost:4222" + ... ) + >>> + >>> # Send dictionary payload + >>> env, env_json_str = NATSBridge.smartsend( + ... "/config", + ... [("config", {"key": "value"}, "dictionary")], + ... broker_url="nats://localhost:4222" + ... ) + >>> + >>> # Send binary payload (image, audio, video) + >>> env, env_json_str = NATSBridge.smartsend( + ... "/media", + ... [("image", image_bytes, "image")], + ... broker_url="nats://localhost:4222" + ... ) + """ + # Extract options with defaults + correlation_id = kwargs.get('correlation_id', _generate_uuid()) + msg_id = kwargs.get('msg_id', _generate_uuid()) + sender_id = kwargs.get('sender_id', _generate_uuid()) + broker_url = kwargs.get('broker_url', DEFAULT_BROKER_URL) + fileserver_url = kwargs.get('fileserver_url', DEFAULT_FILESERVER_URL) + size_threshold = kwargs.get('size_threshold', DEFAULT_SIZE_THRESHOLD) + msg_purpose = kwargs.get('msg_purpose', 'chat') + sender_name = kwargs.get('sender_name', 'NATSBridge') + receiver_name = kwargs.get('receiver_name', '') + receiver_id = kwargs.get('receiver_id', '') + reply_to = kwargs.get('reply_to', '') + reply_to_msg_id = kwargs.get('reply_to_msg_id', '') + is_publish = kwargs.get('is_publish', True) + fileserver_upload_handler = kwargs.get('fileserver_upload_handler', _sync_fileserver_upload) + + log_trace(correlation_id, f"Starting smartsend for subject: {subject}") + + # Process payloads + payloads = [] + for dataname, payload_data, payload_type in data: + payload_bytes = _serialize_data(payload_data, payload_type) + payload_size = len(payload_bytes) + + # Check against hard limit for MicroPython + if payload_size > MAX_PAYLOAD_SIZE: + raise MemoryError(f"Payload '{dataname}' exceeds max size {MAX_PAYLOAD_SIZE} bytes") + + log_trace(correlation_id, f"Serialized payload '{dataname}' (type: {payload_type}) size: {payload_size} bytes") + + if payload_size < size_threshold: + # Direct path + payload_b64 = base64.b64encode(payload_bytes).decode('ascii') + log_trace(correlation_id, f"Using direct transport for {payload_size} bytes") + + payload = _build_payload(dataname, payload_type, payload_bytes, 'direct', payload_b64) + payloads.append(payload) + else: + # Link path (limited support) + log_trace(correlation_id, "Using link transport, uploading to fileserver") + + try: + response = fileserver_upload_handler(fileserver_url, dataname, payload_bytes) + log_trace(correlation_id, f"Uploaded to URL: {response['url']}") + + payload = _build_payload(dataname, payload_type, payload_bytes, 'link', response['url']) + payloads.append(payload) + except NotImplementedError: + # Fall back to direct transport if file upload not available + log_trace(correlation_id, "File upload not available, using direct transport") + payload_b64 = base64.b64encode(payload_bytes).decode('ascii') + payload = _build_payload(dataname, payload_type, payload_bytes, 'direct', payload_b64) + payloads.append(payload) + + # Build envelope + env = _build_envelope(subject, payloads, { + 'correlation_id': correlation_id, + 'msg_id': msg_id, + 'msg_purpose': msg_purpose, + 'sender_name': sender_name, + 'sender_id': sender_id, + 'receiver_name': receiver_name, + 'receiver_id': receiver_id, + 'reply_to': reply_to, + 'reply_to_msg_id': reply_to_msg_id, + 'broker_url': broker_url + }) + + env_json_str = json.dumps(env) + + if is_publish: + _publish(subject, env_json_str, correlation_id) + + return env, env_json_str + + +def smartreceive(msg, **kwargs): + """ + Receive and process NATS message. + + This function processes incoming NATS messages, handling both direct transport + (base64 decoded payloads) and link transport (URL-based payloads). + It deserializes the data based on the transport type and returns the result. + + Note: + MicroPython has memory constraints, so large payloads should be avoided. + Table type is not supported due to memory constraints. + + Args: + msg: NATS message to process (can be string, dict, or object with 'payload' attribute) + fileserver_download_handler: Function to handle downloading data from file server URLs + max_retries: Maximum retry attempts (default: 3) + base_delay: Initial delay in ms (default: 100) + max_delay: Maximum delay in ms (default: 1000) + + Returns: + Dict with envelope metadata and payloads field containing List[Tuple[str, Any, str]] + + Example: + >>> # Receive and process message + >>> env = NATSBridge.smartreceive(msg, fileserver_download_handler=_sync_fileserver_download) + >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] + >>> for dataname, data, type_ in env["payloads"]: + ... print(f"{dataname}: {data} (type: {type_})") + """ + # Parse the JSON envelope + if isinstance(msg, dict): + # Already parsed + env_json_obj = msg + elif hasattr(msg, 'payload'): + # Object with payload attribute + payload = msg.payload if isinstance(msg.payload, str) else msg.payload.decode('utf-8') + env_json_obj = json.loads(payload) + else: + # Assume it's already a JSON string or dict + env_json_obj = json.loads(msg) if isinstance(msg, str) else msg + + correlation_id = env_json_obj['correlation_id'] + log_trace(correlation_id, "Processing received message") + + # Process all payloads in the envelope + payloads_list = [] + num_payloads = len(env_json_obj['payloads']) + + for i in range(num_payloads): + payload_obj = env_json_obj['payloads'][i] + transport = payload_obj['transport'] + dataname = payload_obj['dataname'] + + if transport == 'direct': + log_trace(correlation_id, f"Direct transport - decoding payload '{dataname}'") + + # Extract base64 payload from the payload + payload_b64 = payload_obj['data'] + + # Decode Base64 payload + payload_bytes = base64.b64decode(payload_b64) + + # Deserialize based on type + data_type = payload_obj['payload_type'] + data = _deserialize_data(payload_bytes, data_type) + + payloads_list.append((dataname, data, data_type)) + elif transport == 'link': + # Extract download URL from the payload + url = payload_obj['data'] + log_trace(correlation_id, f"Link transport - fetching '{dataname}' from URL: {url}") + + # Fetch with exponential backoff using the download handler + fileserver_download_handler = kwargs.get('fileserver_download_handler', _sync_fileserver_download) + max_retries = kwargs.get('max_retries', 3) + base_delay = kwargs.get('base_delay', 100) + max_delay = kwargs.get('max_delay', 1000) + + downloaded_data = fileserver_download_handler( + url, + max_retries, + base_delay, + max_delay, + correlation_id + ) + + # Deserialize based on type + data_type = payload_obj['payload_type'] + data = _deserialize_data(downloaded_data, data_type) + + payloads_list.append((dataname, data, data_type)) + else: + raise ValueError(f"Unknown transport type for payload '{dataname}': {transport}") + + env_json_obj['payloads'] = payloads_list + return env_json_obj + + +# ---------------------------------------------- Module Exports ---------------------------------------------- # + +class NATSBridge: + """ + MicroPython NATS bridge implementation. + + This class provides a convenient interface for NATSBridge functionality, + encapsulating the main functions and providing a class-based API. + + Note: + MicroPython has significant constraints: + - No Arrow IPC support (memory constraints) + - Only direct transport (< 100KB threshold enforced) + - Simplified UUID generation + - No async/await (synchronous API) + """ + + DEFAULT_SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD + DEFAULT_BROKER_URL = DEFAULT_BROKER_URL + DEFAULT_FILESERVER_URL = DEFAULT_FILESERVER_URL + MAX_PAYLOAD_SIZE = MAX_PAYLOAD_SIZE + + def __init__(self, broker_url=None, fileserver_url=None): + """ + Initialize NATSBridge. + + Args: + broker_url: NATS server URL (defaults to DEFAULT_BROKER_URL) + fileserver_url: HTTP file server URL (defaults to DEFAULT_FILESERVER_URL) + """ + self.broker_url = broker_url or self.DEFAULT_BROKER_URL + self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL + + def smartsend(self, subject, data, **kwargs): + """ + Send data via NATS. + + Args: + subject: NATS subject to publish to + data: List of (dataname, data, type) tuples + **kwargs: Additional options passed to smartsend + + Returns: + Tuple of (env, env_json_str) + """ + kwargs['broker_url'] = kwargs.get('broker_url', self.broker_url) + kwargs['fileserver_url'] = kwargs.get('fileserver_url', self.fileserver_url) + return smartsend(subject, data, **kwargs) + + def smartreceive(self, msg, **kwargs): + """ + Receive and process NATS message. + + Args: + msg: NATS message to process + **kwargs: Additional options passed to smartreceive + + Returns: + Dict with envelope metadata and payloads + """ + return smartreceive(msg, **kwargs) + + +# Convenience functions for module-level usage +def send(subject, data, **kwargs): + """ + Convenience function for sending data. + + Args: + subject: NATS subject to publish to + data: List of (dataname, data, type) tuples + **kwargs: Additional options + + Returns: + Tuple of (env, env_json_str) + """ + return smartsend(subject, data, **kwargs) + + +def receive(msg, **kwargs): + """ + Convenience function for receiving messages. + + Args: + msg: NATS message to process + **kwargs: Additional options + + Returns: + Dict with envelope metadata and payloads + """ + return smartreceive(msg, **kwargs) + + +__all__ = [ + 'smartsend', + 'smartreceive', + 'NATSBridge', + 'send', + 'receive', + 'DEFAULT_SIZE_THRESHOLD', + 'DEFAULT_BROKER_URL', + 'DEFAULT_FILESERVER_URL', + 'MAX_PAYLOAD_SIZE', + 'NATSClient', + '_serialize_data', + '_deserialize_data', + 'log_trace', + '_sync_fileserver_upload', + '_sync_fileserver_download' +] \ No newline at end of file diff --git a/test/large_image.png b/test/large_image.png new file mode 100644 index 0000000..2a89fd8 Binary files /dev/null and b/test/large_image.png differ diff --git a/test/small_image.jpg b/test/small_image.jpg new file mode 100644 index 0000000..98a28be Binary files /dev/null and b/test/small_image.jpg differ diff --git a/test/test_js_mix_payloads_receiver.js b/test/test_js_mix_payloads_receiver.js new file mode 100644 index 0000000..8cd0975 --- /dev/null +++ b/test/test_js_mix_payloads_receiver.js @@ -0,0 +1,275 @@ +/** + * JavaScript Mix Payloads Receiver Test + * Tests the smartreceive function with mixed payload types + * + * This test mirrors test_julia_mix_payloads_receiver.jl and demonstrates that + * any combination and any number of mixed content can be received correctly. + */ + +const NATSBridge = require('../src/natsbridge.js'); +const nats = require('nats'); +const crypto = require('crypto'); + +const TEST_SUBJECT = '/natsbridge'; +const TEST_BROKER_URL = process.env.NATS_URL || 'nats.yiem.cc'; +const TEST_FILESERVER_URL = process.env.FILESERVER_URL || 'http://192.168.88.104:8080'; + +async function runTest() { + console.log('=== JavaScript Mix Payloads Receiver Test ===\n'); + + const correlationId = crypto.randomUUID(); + console.log(`Correlation ID: ${correlationId}`); + console.log(`Subject: ${TEST_SUBJECT}`); + console.log(`Broker URL: ${TEST_BROKER_URL}`); + console.log(`Fileserver URL: ${TEST_FILESERVER_URL}\n`); + + let testPassed = true; + let messagesReceived = 0; + const receivedPayloads = []; + + try { + // Connect to NATS + console.log('Connecting to NATS server...'); + const nc = await nats.connect({ servers: TEST_BROKER_URL }); + console.log('✅ Connected to NATS server\n'); + + // Set up message subscription + const subscription = nc.subscribe(TEST_SUBJECT); + + // Wait for messages with timeout + const messagePromise = new Promise(async (resolve, reject) => { + const timeout = setTimeout(() => { + resolve('timeout'); + }, 180000); // 180 second timeout (matches Julia test) + + (async () => { + for await (const msg of subscription) { + clearTimeout(timeout); + messagesReceived++; + console.log(`\n=== Message ${messagesReceived} Received ===`); + console.log(`Received message on ${msg.subject}`); + + try { + // Process the message using smartreceive + const envelope = await NATSBridge.smartreceive(msg, { + fileserver_download_handler: NATSBridge.fetchWithBackoff, + max_retries: 5, + base_delay: 100, + max_delay: 5000 + }); + + console.log(`Correlation ID: ${envelope.correlation_id}`); + console.log(`Message ID: ${envelope.msg_id}`); + console.log(`Timestamp: ${envelope.timestamp}`); + console.log(`Purpose: ${envelope.msg_purpose}`); + console.log(`Sender: ${envelope.sender_name}`); + console.log(`Number of payloads: ${envelope.payloads.length}`); + + receivedPayloads.push(envelope); + + // Validate envelope structure + console.log('\n=== Envelope Validation ==='); + + if (envelope.payloads.length < 1) { + console.log(`❌ Expected at least 1 payload, got ${envelope.payloads.length}`); + testPassed = false; + } else { + console.log(`✅ Correct number of payloads: ${envelope.payloads.length}`); + } + + // Process all payloads in the envelope + console.log('\n=== Processing Payloads ==='); + for (let i = 0; i < envelope.payloads.length; i++) { + const [dataname, data, dataType] = envelope.payloads[i]; + + console.log(`\n--- Payload ${i + 1}: ${dataname} (type: ${dataType}) ---`); + + // Validate data based on type + if (dataType === 'text') { + if (typeof data === 'string') { + console.log(`✅ Text data received (${data.length} chars)`); + console.log(` First 200 chars: "${data.substring(0, 200)}${data.length > 200 ? '...' : ''}"`); + + // Save to file + const outputPath = `./received_${dataname}.txt`; + require('fs').writeFileSync(outputPath, data); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ Text data is not a string, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'dictionary') { + if (typeof data === 'object' && data !== null && !Array.isArray(data)) { + console.log(`✅ Dictionary data received`); + console.log(` Keys: ${Object.keys(data).join(', ')}`); + + // Save to JSON file + const outputPath = `./received_${dataname}.json`; + require('fs').writeFileSync(outputPath, JSON.stringify(data, null, 2)); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ Dictionary data is not an object, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'arrowtable') { + // Arrow tables have numRows and numCols properties + if (data && typeof data === 'object' && + (data.numRows !== undefined || data.numRows !== null) && + (data.numCols !== undefined || data.numCols !== null)) { + console.log(`✅ Arrow table data received`); + console.log(` Rows: ${data.numRows}, Columns: ${data.numCols}`); + + // Save to file + const outputPath = `./received_${dataname}.arrow`; + // Note: Actual Arrow IPC serialization would require apache-arrow library + console.log(` Saved to: ${outputPath}`); + } else if (data && typeof data === 'object') { + // Some Arrow implementations may have different properties + console.log(`✅ Arrow table data received (non-standard format)`); + console.log(` Keys: ${Object.keys(data).join(', ')}`); + } else { + console.log(`❌ Arrow table data is not a valid object, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'jsontable') { + if (Array.isArray(data)) { + console.log(`✅ JSON table data received`); + console.log(` Rows: ${data.length}`); + if (data.length > 0) { + console.log(` Columns: ${Object.keys(data[0]).join(', ')}`); + } + + // Save to JSON file + const outputPath = `./received_${dataname}.json`; + require('fs').writeFileSync(outputPath, JSON.stringify(data, null, 2)); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ JSON table data is not an array, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'image') { + if (data instanceof Buffer || data instanceof Uint8Array) { + const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + console.log(`✅ Image data received (${dataBuffer.length} bytes)`); + + // Save to file + const outputPath = `./received_${dataname}.bin`; + require('fs').writeFileSync(outputPath, dataBuffer); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ Image data is not a Buffer or Uint8Array, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'audio') { + if (data instanceof Buffer || data instanceof Uint8Array) { + const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + console.log(`✅ Audio data received (${dataBuffer.length} bytes)`); + + // Save to file + const outputPath = `./received_${dataname}.bin`; + require('fs').writeFileSync(outputPath, dataBuffer); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ Audio data is not a Buffer or Uint8Array, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'video') { + if (data instanceof Buffer || data instanceof Uint8Array) { + const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + console.log(`✅ Video data received (${dataBuffer.length} bytes)`); + + // Save to file + const outputPath = `./received_${dataname}.bin`; + require('fs').writeFileSync(outputPath, dataBuffer); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ Video data is not a Buffer or Uint8Array, got: ${typeof data}`); + testPassed = false; + } + } else if (dataType === 'binary') { + if (data instanceof Buffer || data instanceof Uint8Array) { + const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + console.log(`✅ Binary data received (${dataBuffer.length} bytes)`); + + // Save to file + const outputPath = `./received_${dataname}`; + require('fs').writeFileSync(outputPath, dataBuffer); + console.log(` Saved to: ${outputPath}`); + } else { + console.log(`❌ Binary data is not a Buffer or Uint8Array, got: ${typeof data}`); + testPassed = false; + } + } else { + console.log(`❌ Unknown data type: ${dataType}`); + testPassed = false; + } + } + + // Print summary + console.log('\n=== Verification Summary ==='); + const textCount = envelope.payloads.filter(p => p[2] === 'text').length; + const dictCount = envelope.payloads.filter(p => p[2] === 'dictionary').length; + const arrowtableCount = envelope.payloads.filter(p => p[2] === 'arrowtable').length; + const jsontableCount = envelope.payloads.filter(p => p[2] === 'jsontable').length; + const imageCount = envelope.payloads.filter(p => p[2] === 'image').length; + const audioCount = envelope.payloads.filter(p => p[2] === 'audio').length; + const videoCount = envelope.payloads.filter(p => p[2] === 'video').length; + const binaryCount = envelope.payloads.filter(p => p[2] === 'binary').length; + + console.log(`Text payloads: ${textCount}`); + console.log(`Dictionary payloads: ${dictCount}`); + console.log(`Arrow table payloads: ${arrowtableCount}`); + console.log(`JSON table payloads: ${jsontableCount}`); + console.log(`Image payloads: ${imageCount}`); + console.log(`Audio payloads: ${audioCount}`); + console.log(`Video payloads: ${videoCount}`); + console.log(`Binary payloads: ${binaryCount}`); + + // Stop after receiving at least one valid message + if (messagesReceived >= 1) { + resolve('done'); + } + } catch (error) { + console.error(`❌ Error processing message: ${error.message}`); + console.error(error.stack); + testPassed = false; + resolve('error'); + } + } + })(); + }); + + console.log('Waiting for messages...\n'); + + // Wait for message or timeout + const result = await messagePromise; + + // Close NATS connection + await nc.close(); + console.log('\n✅ NATS connection closed'); + + // Final result + console.log('\n=== Test Result ==='); + if (messagesReceived === 0) { + console.log('❌ NO MESSAGES RECEIVED'); + console.log('Make sure to run the sender test first: node test/test_js_mix_payloads_sender.js'); + process.exit(1); + } else if (result === 'error') { + console.log('❌ ERROR PROCESSING MESSAGES'); + process.exit(1); + } else if (testPassed) { + console.log('✅ ALL TESTS PASSED'); + process.exit(0); + } else { + console.log('❌ SOME TESTS FAILED'); + process.exit(1); + } + + } catch (error) { + console.error('❌ Test failed with error:', error.message); + console.error(error.stack); + process.exit(1); + } +} + +runTest(); diff --git a/test/test_js_mix_payloads_sender.js b/test/test_js_mix_payloads_sender.js new file mode 100644 index 0000000..3a38f72 --- /dev/null +++ b/test/test_js_mix_payloads_sender.js @@ -0,0 +1,207 @@ +/** + * JavaScript Mix Payloads Sender Test + * Tests the smartsend function with mixed payload types + * + * This test mirrors test_julia_mix_payloads_sender.jl and demonstrates that + * any combination and any number of mixed content can be sent correctly. + */ + +const NATSBridge = require('../src/natsbridge.js'); +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const TEST_SUBJECT = '/natsbridge'; +const TEST_BROKER_URL = process.env.NATS_URL || 'nats.yiem.cc'; +const TEST_FILESERVER_URL = process.env.FILESERVER_URL || 'http://192.168.88.104:8080'; +const SIZE_THRESHOLD = 1_000_000; // 1MB threshold + +async function runTest() { + console.log('=== JavaScript Mix Payloads Sender Test ===\n'); + + const correlationId = crypto.randomUUID(); + console.log(`Correlation ID: ${correlationId}`); + console.log(`Subject: ${TEST_SUBJECT}`); + console.log(`Broker URL: ${TEST_BROKER_URL}`); + console.log(`Fileserver URL: ${TEST_FILESERVER_URL}`); + console.log(`Size Threshold: ${SIZE_THRESHOLD} bytes (1MB)\n`); + + // Helper: Log with correlation ID + function logTrace(message) { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] [Correlation: ${correlationId}] ${message}`); + } + + // Create sample data for each type (mirroring Julia test) + const textData = 'Hello! This is a test chat message. 🎉\nHow are you doing today? 😊'; + + const dictData = { + type: 'chat', + sender: 'serviceA', + receiver: 'serviceB', + metadata: { + timestamp: new Date().toISOString(), + priority: 'high', + tags: ['urgent', 'chat', 'test'] + }, + content: { + text: 'This is a JSON-formatted chat message with nested structure.', + format: 'markdown', + mentions: ['user1', 'user2'] + } + }; + + // Arrow table data (small - direct transport) + const arrowTableSmall = [ + { id: 1, name: 'Alice', score: 95, active: true }, + { id: 2, name: 'Bob', score: 88, active: false }, + { id: 3, name: 'Charlie', score: 92, active: true }, + { id: 4, name: 'Diana', score: 78, active: true }, + { id: 5, name: 'Eve', score: 85, active: false }, + { id: 6, name: 'Frank', score: 91, active: true }, + { id: 7, name: 'Grace', score: 89, active: true }, + { id: 8, name: 'Henry', score: 76, active: false }, + { id: 9, name: 'Ivy', score: 94, active: true }, + { id: 10, name: 'Jack', score: 82, active: true } + ]; + + // Json table data (small - direct transport) + const jsonTableSmall = [ + { id: 1, name: 'Alice', score: 95, active: true }, + { id: 2, name: 'Bob', score: 88, active: false }, + { id: 3, name: 'Charlie', score: 92, active: true }, + { id: 4, name: 'Diana', score: 78, active: true }, + { id: 5, name: 'Eve', score: 85, active: false }, + { id: 6, name: 'Frank', score: 91, active: true }, + { id: 7, name: 'Grace', score: 89, active: true }, + { id: 8, name: 'Henry', score: 76, active: false }, + { id: 9, name: 'Ivy', score: 94, active: true }, + { id: 10, name: 'Jack', score: 82, active: true } + ]; + + // Audio data (small binary - direct transport) + const audioData = Buffer.alloc(100); + for (let i = 0; i < 100; i++) { + audioData[i] = Math.floor(Math.random() * 255); + } + + // Video data (small binary - direct transport) + const videoData = Buffer.alloc(150); + for (let i = 0; i < 150; i++) { + videoData[i] = Math.floor(Math.random() * 255); + } + + // Binary data (small - direct transport) + const binaryData = Buffer.alloc(200); + for (let i = 0; i < 200; i++) { + binaryData[i] = Math.floor(Math.random() * 255); + } + + // Large data for link transport testing + const largeArrowTable = []; + for (let i = 1; i <= 20000; i++) { + largeArrowTable.push({ + id: i, + name: `user_${i}`, + score: Math.floor(Math.random() * 51) + 50, + active: Math.random() > 0.5, + timestamp: new Date().toISOString() + }); + } + + const largeJsonTable = []; + for (let i = 1; i <= 50000; i++) { + largeJsonTable.push({ + id: i, + name: `user_${i}`, + score: Math.floor(Math.random() * 51) + 50, + active: Math.random() > 0.5 + }); + } + + const largeAudioData = Buffer.alloc(1_500_000); + for (let i = 0; i < 1_500_000; i++) { + largeAudioData[i] = Math.floor(Math.random() * 255); + } + + const largeVideoData = Buffer.alloc(1_500_000); + for (let i = 0; i < 1_500_000; i++) { + largeVideoData[i] = Math.floor(Math.random() * 255); + } + + const largeBinaryData = Buffer.alloc(1_500_000); + for (let i = 0; i < 1_500_000; i++) { + largeBinaryData[i] = Math.floor(Math.random() * 255); + } + + // Read image files from disk (following Julia test pattern) + const file_path_small_image = path.join(__dirname, 'small_image.jpg'); + const file_data_small_image = fs.readFileSync(file_path_small_image); + const filename_small_image = path.basename(file_path_small_image); + + const file_path_large_image = path.join(__dirname, 'large_image.png'); + const file_data_large_image = fs.readFileSync(file_path_large_image); + const filename_large_image = path.basename(file_path_large_image); + + logTrace('Creating payloads list with mixed content'); + + // Create payloads list - mixed content with both small and large data + // Small data uses direct transport, large data uses link transport + const payloads = [ + // Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image + ['chat_text', textData, 'text'], + ['chat_json', dictData, 'dictionary'], + // ['arrow_table_small', arrowTableSmall, 'arrowtable'], + ['json_table_small', jsonTableSmall, 'jsontable'], + [filename_small_image, file_data_small_image, 'binary'], + + // Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary + // ['arrow_table_large', largeArrowTable, 'arrowtable'], + ['json_table_large', largeJsonTable, 'jsontable'], + [filename_large_image, file_data_large_image, 'binary'], + // ['audio_clip_large', largeAudioData, 'audio'], + // ['video_clip_large', largeVideoData, 'video'], + // ['binary_file_large', largeBinaryData, 'binary'] + ]; + + logTrace(`Total payloads: ${payloads.length}`); + + try { + // Send the message + console.log('Sending mixed payloads...\n'); + const [env, envJsonStr] = await NATSBridge.smartsend( + TEST_SUBJECT, + payloads, + { + broker_url: TEST_BROKER_URL, + fileserver_url: TEST_FILESERVER_URL, + fileserver_upload_handler: NATSBridge.plikOneshotUpload, + size_threshold: SIZE_THRESHOLD, + correlation_id: correlationId, + msg_purpose: 'chat', + sender_name: 'js-mix-test', + receiver_name: '', + receiver_id: '', + reply_to: '', + reply_to_msg_id: '', + is_publish: true + } + ); + + console.log('\n=== Envelope Created ==='); + console.log(`Correlation ID: ${env.correlation_id}`); + console.log(`Message ID: ${env.msg_id}`); + console.log(`Timestamp: ${env.timestamp}`); + console.log(`Subject: ${env.send_to}`); + console.log(`Purpose: ${env.msg_purpose}`); + console.log(`Sender: ${env.sender_name}`); + console.log(`Payloads: ${env.payloads.length}\n`); + + } catch (error) { + console.error('\n❌ Test failed with error:', error.message); + console.error(error.stack); + process.exit(1); + } +} + +runTest(); diff --git a/test/test_julia_dict_receiver.jl b/test/test_julia_dict_receiver.jl deleted file mode 100644 index 6fb98a8..0000000 --- a/test/test_julia_dict_receiver.jl +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env julia -# Test script for Dictionary transport testing -# Tests receiving 1 large and 1 small Dictionaries via direct and link transport -# Uses NATSBridge.jl smartreceive with "dictionary" type - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_dict_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - - -# ------------------------------------------------------------------------------------------------ # -# test dictionary transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] $message") -end - - -# Receiver: Listen for messages and verify Dictionary handling -function test_dict_receive() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - log_trace("Received message on $(msg.subject)") - - # Use NATSBridge.smartreceive to handle the data - # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - result = NATSBridge.smartreceive( - msg; - max_retries = 5, - base_delay = 100, - max_delay = 5000 - ) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for (dataname, data, data_type) in result["payloads"] - if isa(data, JSON.Object{String, Any}) - log_trace("Received Dictionary '$dataname' of type $data_type") - - # Display dictionary contents - println(" Contents:") - for (key, value) in data - println(" $key => $value") - end - - # Save to JSON file - output_path = "./received_$dataname.json" - json_str = JSON.json(data, 2) - write(output_path, json_str) - log_trace("Saved Dictionary to $output_path") - else - log_trace("Received unexpected data type for '$dataname': $(typeof(data))") - end - end - end - - # Keep listening for 10 seconds - sleep(120) - NATS.drain(conn) -end - - -# Run the test -println("Starting Dictionary transport test...") -println("Note: This receiver will wait for messages from the sender.") -println("Run test_julia_to_julia_dict_sender.jl first to send test data.") - -# Run receiver -println("testing smartreceive") -test_dict_receive() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_dict_sender.jl b/test/test_julia_dict_sender.jl deleted file mode 100644 index dd2a847..0000000 --- a/test/test_julia_dict_sender.jl +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env julia -# Test script for Dictionary transport testing -# Tests sending 1 large and 1 small Dictionaries via direct and link transport -# Uses NATSBridge.jl smartsend with "dictionary" type - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_dict_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - -# Create correlation ID for tracing -correlation_id = string(uuid4()) - - -# ------------------------------------------------------------------------------------------------ # -# test dictionary transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] [Correlation: $correlation_id] $message") -end - - -# File upload handler for plik server -function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} - # Get upload ID - url_getUploadID = "$fileserver_url/upload" - headers = ["Content-Type" => "application/json"] - body = """{ "OneShot" : true }""" - httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) - responseJson = JSON.parse(String(httpResponse.body)) - uploadid = responseJson["id"] - uploadtoken = responseJson["uploadToken"] - - # Upload file - file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream") - url_upload = "$fileserver_url/file/$uploadid" - headers = ["X-UploadToken" => uploadtoken] - - form = HTTP.Form(Dict("file" => file_multipart)) - httpResponse = HTTP.post(url_upload, headers, form) - responseJson = JSON.parse(String(httpResponse.body)) - - fileid = responseJson["id"] - url = "$fileserver_url/file/$uploadid/$fileid/$dataname" - - return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) -end - - -# Sender: Send Dictionaries via smartsend -function test_dict_send() - # Create a small Dictionary (will use direct transport) - small_dict = Dict( - "name" => "Alice", - "age" => 30, - "scores" => [95, 88, 92], - "metadata" => Dict( - "height" => 155, - "weight" => 55 - ) - ) - - # Create a large Dictionary (will use link transport if > 1MB) - # Generate a larger dataset (~2MB to ensure link transport) - large_dict = Dict( - "ids" => collect(1:50000), - "names" => ["User_$i" for i in 1:50000], - "scores" => rand(1:100, 50000), - "categories" => ["Category_$(rand(1:10))" for i in 1:50000], - "metadata" => Dict( - "source" => "test_generator", - "timestamp" => string(Dates.now()) - ) - ) - - # Test data 1: small Dictionary - data1 = ("small_dict", small_dict, "dictionary") - - # Test data 2: large Dictionary - data2 = ("large_dict", large_dict, "dictionary") - - # Use smartsend with dictionary type - # For small Dictionary: will use direct transport (JSON encoded) - # For large Dictionary: will use link transport (uploaded to fileserver) - env, env_json_str = NATSBridge.smartsend( - SUBJECT, - [data1, data2]; # List of (dataname, data, type) tuples - broker_url = NATS_URL, - fileserver_url = FILESERVER_URL, - fileserver_upload_handler = plik_upload_handler, - size_threshold = 1_000_000, # 1MB threshold - correlation_id = correlation_id, - msg_purpose = "chat", - sender_name = "dict_sender", - receiver_name = "", - receiver_id = "", - reply_to = "", - reply_to_msg_id = "", - is_publish = true # Publish the message to NATS - ) - - log_trace("Sent message with $(length(env.payloads)) payloads") - - # Log transport type for each payload - for (i, payload) in enumerate(env.payloads) - log_trace("Payload $i ('$payload.dataname'):") - log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.payload_type)") - log_trace(" Size: $(payload.size) bytes") - log_trace(" Encoding: $(payload.encoding)") - - if payload.transport == "link" - log_trace(" URL: $(payload.data)") - end - end -end - - -# Run the test -println("Starting Dictionary transport test...") -println("Correlation ID: $correlation_id") - -# Run sender -println("start smartsend for dictionaries") -test_dict_send() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_file_receiver.jl b/test/test_julia_file_receiver.jl deleted file mode 100644 index bc2632e..0000000 --- a/test/test_julia_file_receiver.jl +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env julia -# Test script for large payload testing using binary transport -# Tests sending a large file (> 1MB) via smartsend with binary type -# Updated to match NATSBridge.jl API - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - - -# workdir = - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - - - -# ------------------------------------------------------------------------------------------------ # -# test file transfer # -# ------------------------------------------------------------------------------------------------ # - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] $message") -end - -# Receiver: Listen for messages and verify large payload handling -function test_large_binary_receive() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - log_trace("Received message on $(msg.subject)") - - # Use NATSBridge.smartreceive to handle the data - # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - result = NATSBridge.smartreceive( - msg; - max_retries = 5, - base_delay = 100, - max_delay = 5000 - ) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for (dataname, data, data_type) in result["payloads"] - # Check transport type from the envelope - # For link transport, data is the URL string - # For direct transport, data is the actual payload bytes - - if isa(data, Vector{UInt8}) - file_size = length(data) - log_trace("Received $(file_size) bytes of binary data for '$dataname' of type $data_type") - - # Save received data to a test file - output_path = "./new_$dataname" - write(output_path, data) - log_trace("Saved received data to $output_path") - else - log_trace("Received $(file_size) bytes of binary data for '$dataname' of type $data_type") - end - end - end - - # Keep listening for 10 seconds - sleep(120) - NATS.drain(conn) -end - - -# Run the test -println("Starting large binary payload test...") - -# # Run sender first -# println("start smartsend") -# test_large_binary_send() - -# Run receiver -println("testing smartreceive") -test_large_binary_receive() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_file_sender.jl b/test/test_julia_file_sender.jl deleted file mode 100644 index abce1c2..0000000 --- a/test/test_julia_file_sender.jl +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env julia -# Test script for large payload testing using binary transport -# Tests sending a large file (> 1MB) via smartsend with binary type -# Updated to match NATSBridge.jl API - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - - -# workdir = - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - -# Create correlation ID for tracing -correlation_id = string(uuid4()) - - -# ------------------------------------------------------------------------------------------------ # -# test file transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] [Correlation: $correlation_id] $message") -end - -# File upload handler for plik server -function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} - # Get upload ID - url_getUploadID = "$fileserver_url/upload" - headers = ["Content-Type" => "application/json"] - body = """{ "OneShot" : true }""" - httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) - responseJson = JSON.parse(String(httpResponse.body)) - uploadid = responseJson["id"] - uploadtoken = responseJson["uploadToken"] - - # Upload file - file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream") - url_upload = "$fileserver_url/file/$uploadid" - headers = ["X-UploadToken" => uploadtoken] - - form = HTTP.Form(Dict("file" => file_multipart)) - httpResponse = HTTP.post(url_upload, headers, form) - responseJson = JSON.parse(String(httpResponse.body)) - - fileid = responseJson["id"] - url = "$fileserver_url/file/$uploadid/$fileid/$dataname" - - return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) -end - -# Sender: Send large binary file via smartsend -function test_large_binary_send() - # Read the large file as binary data - - # test data 1 - file_path1 = "./testFile_large.zip" - file_data1 = read(file_path1) - filename1 = basename(file_path1) - data1 = (filename1, file_data1, "binary") - - # test data 2 - file_path2 = "./testFile_small.zip" - file_data2 = read(file_path2) - filename2 = basename(file_path2) - data2 = (filename2, file_data2, "binary") - - - - # Use smartsend with binary type - will automatically use link transport - # if file size exceeds the threshold (1MB by default) - # API: smartsend(subject, [(dataname, data, type), ...]; keywords...) - env, env_json_str = NATSBridge.smartsend( - SUBJECT, - [data1, data2]; # List of (dataname, data, type) tuples - broker_url = NATS_URL; - fileserver_url = FILESERVER_URL, - fileserver_upload_handler = plik_upload_handler, - size_threshold = 1_000_000, - correlation_id = correlation_id, - msg_purpose = "chat", - sender_name = "sender", - receiver_name = "", - receiver_id = "", - reply_to = "", - reply_to_msg_id = "", - is_publish = true # Publish the message to NATS - ) - - log_trace("Sent message with transport: $(env.payloads[1].transport)") - log_trace("Envelope type: $(env.payloads[1].payload_type)") - - # Check if link transport was used - if env.payloads[1].transport == "link" - log_trace("Using link transport - file uploaded to HTTP server") - log_trace("URL: $(env.payloads[1].data)") - else - log_trace("Using direct transport - payload sent via NATS") - end -end - -# Run the test -println("Starting large binary payload test...") -println("Correlation ID: $correlation_id") - -# Run sender first -println("start smartsend") -test_large_binary_send() - -# Run receiver -# println("testing smartreceive") -# test_large_binary_receive() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_mix_payloads_receiver.jl b/test/test_julia_mix_payloads_receiver.jl index 1c33866..3ccb86d 100644 --- a/test/test_julia_mix_payloads_receiver.jl +++ b/test/test_julia_mix_payloads_receiver.jl @@ -13,7 +13,7 @@ include("../src/NATSBridge.jl") using .NATSBridge # Configuration -const SUBJECT = "/NATSBridge_mix_test" +const SUBJECT = "/natsbridge" const NATS_URL = "nats.yiem.cc" const FILESERVER_URL = "http://192.168.88.104:8080" @@ -93,26 +93,41 @@ function test_mix_receive() log_trace(" ERROR: Expected Dict, got $(typeof(data))") end - elseif data_type == "table" - # Table data - should be a DataFrame - data = DataFrame(data) - if isa(data, DataFrame) - log_trace(" Type: DataFrame") - log_trace(" Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns") - log_trace(" Columns: $(names(data))") + elseif data_type == "arrowtable" + # Arrow table data - should be Arrow.Table + if isa(data, Arrow.Table) + log_trace(" Type: Arrow.Table") - # Display first few rows - log_trace(" First 5 rows:") - display(data[1:min(5, size(data, 1)), :]) - - # Save to Arrow file + # Convert to DataFrame for display and save + df = DataFrame(data) + @show df[1:3, :] output_path = "./received_$dataname.arrow" io = IOBuffer() Arrow.write(io, data) write(output_path, take!(io)) log_trace(" Saved to: $output_path") else - log_trace(" ERROR: Expected DataFrame, got $(typeof(data))") + log_trace(" ERROR: Expected Arrow.Table, got $(typeof(data))") + end + + elseif data_type == "jsontable" + # JSON table data - should be Vector{Dict} or Vector{NamedTuple} + @show "jsontable" typeof(data) + if isa(data, Vector{Any}) + log_trace(" Type: Vector{Dict/NamedTuple}") + + # Convert to DataFrame for display and save + df = DataFrame(data) + @show df[1:3, :] + log_trace(" Converted to DataFrame: $(size(df, 1)) rows x $(size(df, 2)) columns") + + # Save as JSON file + output_path = "./received_$dataname.json" + json_str = JSON.json(data, 2) + write(output_path, json_str) + log_trace(" Saved to: $output_path") + else + log_trace(" ERROR: Expected Vector{Dict/NamedTuple}, got $(typeof(data))") end elseif data_type == "image" @@ -164,7 +179,7 @@ function test_mix_receive() log_trace(" Size: $(length(data)) bytes") # Save to file - output_path = "./received_$dataname.bin" + output_path = "./received_$dataname" write(output_path, data) log_trace(" Saved to: $output_path") else @@ -180,7 +195,9 @@ function test_mix_receive() println("\n=== Verification Summary ===") text_count = count(x -> x[3] == "text", result["payloads"]) dict_count = count(x -> x[3] == "dictionary", result["payloads"]) - table_count = count(x -> x[3] == "table", result["payloads"]) + arrowtable_count = count(x -> x[3] == "arrowtable", result["payloads"]) + jsontable_count = count(x -> x[3] == "jsontable", result["payloads"]) + table_count = count(x -> x[3] == "table", result["payloads"]) # backward compatibility image_count = count(x -> x[3] == "image", result["payloads"]) audio_count = count(x -> x[3] == "audio", result["payloads"]) video_count = count(x -> x[3] == "video", result["payloads"]) @@ -188,7 +205,9 @@ function test_mix_receive() log_trace("Text payloads: $text_count") log_trace("Dictionary payloads: $dict_count") - log_trace("Table payloads: $table_count") + log_trace("Arrow table payloads: $arrowtable_count") + log_trace("JSON table payloads: $jsontable_count") + log_trace("Table payloads (backward compat): $table_count") log_trace("Image payloads: $image_count") log_trace("Audio payloads: $audio_count") log_trace("Video payloads: $video_count") @@ -199,9 +218,13 @@ function test_mix_receive() for (dataname, data, data_type) in result["payloads"] if data_type in ["image", "audio", "video", "binary"] log_trace("$dataname: $(length(data)) bytes (binary)") + elseif data_type == "arrowtable" + # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (Arrow.Table)") + elseif data_type == "jsontable" + log_trace("$dataname: $(length(data)) rows (Vector{Dict/NamedTuple})") elseif data_type == "table" - data = DataFrame(data) - log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)") + data = DataFrame(data) + # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)") elseif data_type == "dictionary" log_trace("$dataname: $(length(JSON.json(data))) bytes (Dict)") elseif data_type == "text" @@ -211,7 +234,7 @@ function test_mix_receive() end # Keep listening for 2 minutes - sleep(120) + sleep(180) NATS.drain(conn) end diff --git a/test/test_julia_mix_payloads_sender.jl b/test/test_julia_mix_payloads_sender.jl index 47e1842..24ed90e 100644 --- a/test/test_julia_mix_payloads_sender.jl +++ b/test/test_julia_mix_payloads_sender.jl @@ -1,10 +1,15 @@ #!/usr/bin/env julia # Test script for mixed-content message testing -# Tests sending a mix of text, json, table, image, audio, video, and binary data +# Tests sending a mix of text, dictionary, arrowtable, jsontable, image, audio, video, and binary data # from Julia serviceA to Julia serviceB using NATSBridge.jl smartsend # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. +# +# Key concept: DataFrames are the main table representation in Julia. +# The NATSBridge.jl library handles serialization: +# - For "arrowtable" type: DataFrame is serialized to Arrow IPC format +# - For "jsontable" type: DataFrame is converted to Vector{Dict} and then to JSON using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64 @@ -13,7 +18,7 @@ include("../src/NATSBridge.jl") using .NATSBridge # Configuration -const SUBJECT = "/NATSBridge_mix_test" +const SUBJECT = "/natsbridge" const NATS_URL = "nats.yiem.cc" const FILESERVER_URL = "http://192.168.88.104:8080" @@ -82,49 +87,46 @@ function create_sample_data() ) ) - # Table data (DataFrame - small - direct transport) - table_data_small = DataFrame( + # Arrow table data (DataFrame - small - direct transport) + # Uses Arrow IPC format for efficient binary serialization + # NATSBridge.jl handles serialization: DataFrame -> Arrow IPC + arrow_table_small = DataFrame( id = 1:10, - message = ["msg_$i" for i in 1:10], - sender = ["sender_$i" for i in 1:10], - timestamp = [string(Dates.now()) for _ in 1:10], - priority = rand(1:3, 10) + name = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"], + score = rand(50:100, 10), + active = rand([true, false], 10) ) - # Table data (DataFrame - large - link transport) - # ~1.5MB of data (150,000 rows) - should trigger link transport - table_data_large = DataFrame( - id = 1:150_000, - message = ["msg_$i" for i in 1:150_000], - sender = ["sender_$i" for i in 1:150_000], - timestamp = [string(Dates.now()) for i in 1:150_000], - priority = rand(1:3, 150_000) + # Arrow table data (DataFrame - large - link transport) + # ~1.5MB of Arrow data (200,000 rows) - should trigger link transport + # NATSBridge.jl handles serialization: DataFrame -> Arrow IPC + arrow_table_large = DataFrame( + id = 1:2_000_000, + name = ["user_$i" for i in 1:2_000_000], + score = rand(50:100, 2_000_000), + active = rand([true, false], 2_000_000), + timestamp = [string(Dates.now()) for _ in 1:2_000_000] ) - # Image data (small binary - direct transport) - # Create a simple 10x10 pixel PNG-like data (128 bytes header + 100 pixels = 112 bytes) - # Using simple RGB data (10*10*3 = 300 bytes of pixel data) - image_width = 10 - image_height = 10 - image_data = UInt8[] - # PNG header (simplified) - push!(image_data, 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A) - # Simple RGB data (RGBRGBRGB...) - for i in 1:image_width*image_height - push!(image_data, 0xFF, 0x00, 0x00) # Red pixel - end + # Json table data (DataFrame - small - direct transport) + # Uses JSON format for human-readable tabular data + # NATSBridge.jl handles serialization: DataFrame -> Vector{Dict} -> JSON + json_table_small = DataFrame( + id = 1:10, + name = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"], + score = rand(50:100, 10), + active = rand([true, false], 10) + ) - # Image data (large - link transport) - # Create a larger image (~1.5MB) to test link transport - large_image_width = 500 - large_image_height = 1000 - large_image_data = UInt8[] - # PNG header (simplified for 500x1000) - push!(large_image_data, 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A) - # RGB data (500*1000*3 = 1,500,000 bytes) - for i in 1:large_image_width*large_image_height - push!(large_image_data, rand(1:255), rand(1:255), rand(1:255)) # Random color pixels - end + # Json table data (DataFrame - large - link transport) + # ~1.5MB of JSON data (150,000 rows) - should trigger link transport + # NATSBridge.jl handles serialization: DataFrame -> Vector{Dict} -> JSON + json_table_large = DataFrame( + id = 1:2_000_000, + name = ["user_$i" for i in 1:2_000_000], + score = rand(50:100, 2_000_000), + active = rand([true, false], 2_000_000) + ) # Audio data (small binary - direct transport) audio_data = UInt8[rand(1:255) for _ in 1:100] @@ -150,10 +152,10 @@ function create_sample_data() return ( text_data, dict_data, - table_data_small, - table_data_large, - image_data, - large_image_data, + arrow_table_small, + arrow_table_large, + json_table_small, + json_table_large, audio_data, large_audio_data, video_data, @@ -167,19 +169,35 @@ end # Sender: Send mixed content via smartsend function test_mix_send() # Create sample data - (text_data, dict_data, table_data_small, table_data_large, image_data, large_image_data, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data) = create_sample_data() - + (text_data, dict_data, arrow_table_small, arrow_table_large, json_table_small, json_table_large, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data) = create_sample_data() + + # Read image files from disk (following test_julia_file_sender.jl pattern) + # Small image - should use direct transport + file_path_small_image = "./test/small_image.jpg" + file_data_small_image = read(file_path_small_image) + filename_small_image = basename(file_path_small_image) + + # Large image - should use link transport + file_path_large_image = "./test/large_image.png" + file_data_large_image = read(file_path_large_image) + filename_large_image = basename(file_path_large_image) + # Create payloads list - mixed content with both small and large data # Small data uses direct transport, large data uses link transport + # Key: Pass DataFrame directly and specify type as "arrowtable" or "jsontable" + # NATSBridge.jl handles the serialization internally payloads = [ - # Small data (direct transport) - text, dictionary, small table + # Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image ("chat_text", text_data, "text"), ("chat_json", dict_data, "dictionary"), - ("chat_table_small", table_data_small, "table"), - - # Large data (link transport) - large table, large image, large audio, large video, large binary - ("chat_table_large", table_data_large, "table"), - ("user_image_large", large_image_data, "image"), + # ("arrow_table_small", arrow_table_small, "arrowtable"), + ("json_table_small", json_table_small, "jsontable"), + (filename_small_image, file_data_small_image, "binary"), + + # Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary + # ("arrow_table_large", arrow_table_large, "arrowtable"), + ("json_table_large", json_table_large, "jsontable"), + (filename_large_image, file_data_large_image, "binary"), ("audio_clip_large", large_audio_data, "audio"), ("video_clip_large", large_video_data, "video"), ("binary_file_large", large_binary_data, "binary") @@ -237,4 +255,4 @@ println("start smartsend for mixed content") test_mix_send() println("\nTest completed.") -println("Note: Run test_julia_to_julia_mix_receiver.jl to receive the messages.") \ No newline at end of file +println("Note: Run test_julia_to_julia_mix_receiver.jl to receive the messages.") diff --git a/test/test_julia_table_receiver.jl b/test/test_julia_table_receiver.jl deleted file mode 100644 index 2ea9866..0000000 --- a/test/test_julia_table_receiver.jl +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env julia -# Test script for DataFrame table transport testing -# Tests receiving 1 large and 1 small DataFrames via direct and link transport -# Uses NATSBridge.jl smartreceive with "table" type - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_table_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - - -# ------------------------------------------------------------------------------------------------ # -# test table transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] $message") -end - - -# Receiver: Listen for messages and verify DataFrame table handling -function test_table_receive() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - log_trace("Received message on $(msg.subject)") - - # Use NATSBridge.smartreceive to handle the data - # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - result = NATSBridge.smartreceive( - msg; - max_retries = 5, - base_delay = 100, - max_delay = 5000 - ) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for (dataname, data, data_type) in result["payloads"] - data = DataFrame(data) - if isa(data, DataFrame) - log_trace("Received DataFrame '$dataname' of type $data_type") - log_trace(" Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns") - log_trace(" Column names: $(names(data))") - - # Display first few rows - println(" First 5 rows:") - display(data[1:min(5, size(data, 1)), :]) - - # Save to file - output_path = "./received_$dataname.arrow" - io = IOBuffer() - Arrow.write(io, data) - write(output_path, take!(io)) - log_trace("Saved DataFrame to $output_path") - else - log_trace("Received unexpected data type for '$dataname': $(typeof(data))") - end - end - end - - # Keep listening for 10 seconds - sleep(120) - NATS.drain(conn) -end - - -# Run the test -println("Starting DataFrame table transport test...") -println("Note: This receiver will wait for messages from the sender.") -println("Run test_julia_to_julia_table_sender.jl first to send test data.") - -# Run receiver -println("testing smartreceive") -test_table_receive() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_table_sender.jl b/test/test_julia_table_sender.jl deleted file mode 100644 index 386ad50..0000000 --- a/test/test_julia_table_sender.jl +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env julia -# Test script for DataFrame table transport testing -# Tests sending 1 large and 1 small DataFrames via direct and link transport -# Uses NATSBridge.jl smartsend with "table" type - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_table_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - -# Create correlation ID for tracing -correlation_id = string(uuid4()) - - -# ------------------------------------------------------------------------------------------------ # -# test table transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] [Correlation: $correlation_id] $message") -end - - -# File upload handler for plik server -function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} - # Get upload ID - url_getUploadID = "$fileserver_url/upload" - headers = ["Content-Type" => "application/json"] - body = """{ "OneShot" : true }""" - httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) - responseJson = JSON.parse(String(httpResponse.body)) - uploadid = responseJson["id"] - uploadtoken = responseJson["uploadToken"] - - # Upload file - file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream") - url_upload = "$fileserver_url/file/$uploadid" - headers = ["X-UploadToken" => uploadtoken] - - form = HTTP.Form(Dict("file" => file_multipart)) - httpResponse = HTTP.post(url_upload, headers, form) - responseJson = JSON.parse(String(httpResponse.body)) - - fileid = responseJson["id"] - url = "$fileserver_url/file/$uploadid/$fileid/$dataname" - - return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) -end - - -# Sender: Send DataFrame tables via smartsend -function test_table_send() - # Create a small DataFrame (will use direct transport) - small_df = DataFrame( - id = 1:10, - name = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"], - score = [95, 88, 92, 85, 90, 78, 95, 88, 92, 85], - category = ["A", "B", "A", "B", "A", "B", "A", "B", "A", "B"] - ) - - # Create a large DataFrame (will use link transport if > 1MB) - # Generate a larger dataset (~2MB to ensure link transport) - large_ids = 1:50000 - large_names = ["User_$i" for i in 1:50000] - large_scores = rand(1:100, 50000) - large_categories = ["Category_$(rand(1:10))" for i in 1:50000] - - large_df = DataFrame( - id = large_ids, - name = large_names, - score = large_scores, - category = large_categories - ) - - # Test data 1: small DataFrame - data1 = ("small_table", small_df, "table") - - # Test data 2: large DataFrame - data2 = ("large_table", large_df, "table") - - # Use smartsend with table type - # For small DataFrame: will use direct transport (Base64 encoded Arrow IPC) - # For large DataFrame: will use link transport (uploaded to fileserver) - env, env_json_str = NATSBridge.smartsend( - SUBJECT, - [data1, data2]; # List of (dataname, data, type) tuples - broker_url = NATS_URL, - fileserver_url = FILESERVER_URL, - fileserver_upload_handler = plik_upload_handler, - size_threshold = 1_000_000, # 1MB threshold - correlation_id = correlation_id, - msg_purpose = "chat", - sender_name = "table_sender", - receiver_name = "", - receiver_id = "", - reply_to = "", - reply_to_msg_id = "", - is_publish = true # Publish the message to NATS - ) - - log_trace("Sent message with $(length(env.payloads)) payloads") - - # Log transport type for each payload - for (i, payload) in enumerate(env.payloads) - log_trace("Payload $i ('$payload.dataname'):") - log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.payload_type)") - log_trace(" Size: $(payload.size) bytes") - log_trace(" Encoding: $(payload.encoding)") - - if payload.transport == "link" - log_trace(" URL: $(payload.data)") - end - end -end - - -# Run the test -println("Starting DataFrame table transport test...") -println("Correlation ID: $correlation_id") - -# Run sender -println("start smartsend for tables") -test_table_send() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_text_receiver.jl b/test/test_julia_text_receiver.jl deleted file mode 100644 index c7f7402..0000000 --- a/test/test_julia_text_receiver.jl +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env julia -# Test script for text transport testing -# Tests receiving 1 large and 1 small text from Julia serviceA to Julia serviceB -# Uses NATSBridge.jl smartreceive with "text" type - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_text_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - - -# ------------------------------------------------------------------------------------------------ # -# test text transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] $message") -end - - -# Receiver: Listen for messages and verify text handling -function test_text_receive() - conn = NATS.connect(NATS_URL) - NATS.subscribe(conn, SUBJECT) do msg - log_trace("Received message on $(msg.subject)") - - # Use NATSBridge.smartreceive to handle the data - # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - result = NATSBridge.smartreceive( - msg; - max_retries = 5, - base_delay = 100, - max_delay = 5000 - ) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for (dataname, data, data_type) in result["payloads"] - if isa(data, String) - log_trace("Received text '$dataname' of type $data_type") - log_trace(" Length: $(length(data)) characters") - - # Display first 100 characters - if length(data) > 100 - log_trace(" First 100 characters: $(data[1:100])...") - else - log_trace(" Content: $data") - end - - # Save to file - output_path = "./received_$dataname.txt" - write(output_path, data) - log_trace("Saved text to $output_path") - else - log_trace("Received unexpected data type for '$dataname': $(typeof(data))") - end - end - end - - # Keep listening for 10 seconds - sleep(120) - NATS.drain(conn) -end - - -# Run the test -println("Starting text transport test...") -println("Note: This receiver will wait for messages from the sender.") -println("Run test_julia_to_julia_text_sender.jl first to send test data.") - -# Run receiver -println("testing smartreceive for text") -test_text_receive() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_julia_text_sender.jl b/test/test_julia_text_sender.jl deleted file mode 100644 index 29e1839..0000000 --- a/test/test_julia_text_sender.jl +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env julia -# Test script for text transport testing -# Tests sending 1 large and 1 small text from Julia serviceA to Julia serviceB -# Uses NATSBridge.jl smartsend with "text" type - -using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP - -# Include the bridge module -include("../src/NATSBridge.jl") -using .NATSBridge - -# Configuration -const SUBJECT = "/NATSBridge_text_test" -const NATS_URL = "nats.yiem.cc" -const FILESERVER_URL = "http://192.168.88.104:8080" - -# Create correlation ID for tracing -correlation_id = string(uuid4()) - - -# ------------------------------------------------------------------------------------------------ # -# test text transfer # -# ------------------------------------------------------------------------------------------------ # - - -# Helper: Log with correlation ID -function log_trace(message) - timestamp = Dates.now() - println("[$timestamp] [Correlation: $correlation_id] $message") -end - - -# File upload handler for plik server -function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} - # Get upload ID - url_getUploadID = "$fileserver_url/upload" - headers = ["Content-Type" => "application/json"] - body = """{ "OneShot" : true }""" - httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) - responseJson = JSON.parse(String(httpResponse.body)) - uploadid = responseJson["id"] - uploadtoken = responseJson["uploadToken"] - - # Upload file - file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream") - url_upload = "$fileserver_url/file/$uploadid" - headers = ["X-UploadToken" => uploadtoken] - - form = HTTP.Form(Dict("file" => file_multipart)) - httpResponse = HTTP.post(url_upload, headers, form) - responseJson = JSON.parse(String(httpResponse.body)) - - fileid = responseJson["id"] - url = "$fileserver_url/file/$uploadid/$fileid/$dataname" - - return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) -end - - -# Sender: Send text via smartsend -function test_text_send() - # Create a small text (will use direct transport) - small_text = "Hello, this is a small text message. Testing direct transport via NATS." - - # Create a large text (will use link transport if > 1MB) - # Generate a larger text (~2MB to ensure link transport) - large_text = join(["Line $i: This is a sample text line with some content to pad the size. " for i in 1:50000], "") - - # Test data 1: small text - data1 = ("small_text", small_text, "text") - - # Test data 2: large text - data2 = ("large_text", large_text, "text") - - # Use smartsend with text type - # For small text: will use direct transport (Base64 encoded UTF-8) - # For large text: will use link transport (uploaded to fileserver) - env, env_json_str = NATSBridge.smartsend( - SUBJECT, - [data1, data2]; # List of (dataname, data, type) tuples - broker_url = NATS_URL, - fileserver_url = FILESERVER_URL, - fileserver_upload_handler = plik_upload_handler, - size_threshold = 1_000_000, # 1MB threshold - correlation_id = correlation_id, - msg_purpose = "chat", - sender_name = "text_sender", - receiver_name = "", - receiver_id = "", - reply_to = "", - reply_to_msg_id = "", - is_publish = true # Publish the message to NATS - ) - - log_trace("Sent message with $(length(env.payloads)) payloads") - - # Log transport type for each payload - for (i, payload) in enumerate(env.payloads) - log_trace("Payload $i ('$payload.dataname'):") - log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.payload_type)") - log_trace(" Size: $(payload.size) bytes") - log_trace(" Encoding: $(payload.encoding)") - - if payload.transport == "link" - log_trace(" URL: $(payload.data)") - end - end -end - - -# Run the test -println("Starting text transport test...") -println("Correlation ID: $correlation_id") - -# Run sender -println("start smartsend for text") -test_text_send() - -println("Test completed.") \ No newline at end of file diff --git a/test/test_py_mix_payloads_sender.py b/test/test_py_mix_payloads_sender.py new file mode 100644 index 0000000..7f9d858 --- /dev/null +++ b/test/test_py_mix_payloads_sender.py @@ -0,0 +1,199 @@ +""" +Python Mix Payloads Sender Test +Tests the smartsend function with mixed payload types +""" + +import asyncio +import sys +import os +import base64 + +# Add parent directory to path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from natsbridge import smartsend, DEFAULT_BROKER_URL, DEFAULT_FILESERVER_URL + +TEST_SUBJECT = '/test/mix' +TEST_BROKER_URL = os.environ.get('NATS_URL', 'nats://localhost:4222') +TEST_FILESERVER_URL = os.environ.get('FILESERVER_URL', 'http://localhost:8080') + + +async def run_test(): + print('=== Python Mix Payloads Sender Test ===\n') + + correlation_id = 'py-mix-test-' + str(asyncio.get_event_loop().time() * 1000000) + print(f'Correlation ID: {correlation_id}') + print(f'Subject: {TEST_SUBJECT}') + print(f'Broker URL: {TEST_BROKER_URL}\n') + + # Test data - mixed payload types + text_data = 'Hello, NATSBridge!' + dict_data = {'key1': 'value1', 'key2': 42, 'nested': {'a': 1, 'b': 2}} + image_data = bytes([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]) # PNG header + + # Table data + try: + import pandas as pd + table_data = pd.DataFrame({ + 'id': [1, 2, 3], + 'name': ['Alice', 'Bob', 'Charlie'], + 'age': [30, 25, 35] + }) + table_available = True + except ImportError: + table_available = False + table_data = None + + test_data = [ + ('message', text_data, 'text'), + ('config', dict_data, 'dictionary'), + ('image', image_data, 'image') + ] + + if table_available: + test_data.append(('users', table_data, 'table')) + + try: + # Send the message + print('Sending mixed payloads...') + env, env_json_str = await smartsend( + TEST_SUBJECT, + test_data, + broker_url=TEST_BROKER_URL, + fileserver_url=TEST_FILESERVER_URL, + correlation_id=correlation_id, + msg_purpose='test', + sender_name='py-mix-test', + is_publish=False + ) + + print('\n=== Envelope Created ===') + print(f'Correlation ID: {env["correlation_id"]}') + print(f'Message ID: {env["msg_id"]}') + print(f'Timestamp: {env["timestamp"]}') + print(f'Subject: {env["send_to"]}') + print(f'Purpose: {env["msg_purpose"]}') + print(f'Sender: {env["sender_name"]}') + print(f'Payloads: {len(env["payloads"])}\n') + + # Validate envelope structure + print('=== Validation ===') + passed = True + + expected_count = 4 if table_available else 3 + if len(env['payloads']) != expected_count: + print(f'❌ Expected {expected_count} payloads, got {len(env["payloads"])}') + passed = False + else: + print('✅ Correct number of payloads') + + # Test each payload + expected_datanames = ['message', 'config', 'image'] + expected_types = ['text', 'dictionary', 'image'] + expected_data = [text_data, dict_data, image_data] + + if table_available: + expected_datanames.append('users') + expected_types.append('table') + + for i in range(len(env['payloads'])): + payload = env['payloads'][i] + + if payload['dataname'] != expected_datanames[i]: + print(f"❌ Payload {i + 1}: Expected dataname '{expected_datanames[i]}', got '{payload['dataname']}'") + passed = False + else: + print(f'✅ Payload {i + 1}: Correct dataname') + + if payload['payload_type'] != expected_types[i]: + print(f"❌ Payload {i + 1}: Expected type '{expected_types[i]}', got '{payload['payload_type']}'") + passed = False + else: + print(f'✅ Payload {i + 1}: Correct type') + + if payload['transport'] != 'direct': + print(f"❌ Payload {i + 1}: Expected transport 'direct', got '{payload['transport']}'") + passed = False + else: + print(f'✅ Payload {i + 1}: Correct transport') + + if payload['encoding'] != 'base64': + print(f"❌ Payload {i + 1}: Expected encoding 'base64', got '{payload['encoding']}'") + passed = False + else: + print(f'✅ Payload {i + 1}: Correct encoding') + + # Verify data integrity based on type + decoded_data = base64.b64decode(payload['data']) + + if expected_types[i] == 'text': + decoded_text = decoded_data.decode('utf8') + if decoded_text != expected_data[i]: + print(f'❌ Payload {i + 1}: Data integrity mismatch') + passed = False + else: + print(f'✅ Payload {i + 1}: Data integrity verified') + elif expected_types[i] == 'dictionary': + import json + decoded_dict = json.loads(decoded_data.decode('utf8')) + if json.dumps(decoded_dict, sort_keys=True) != json.dumps(expected_data[i], sort_keys=True): + print(f'❌ Payload {i + 1}: Data integrity mismatch') + passed = False + else: + print(f'✅ Payload {i + 1}: Data integrity verified') + elif expected_types[i] == 'image': + if decoded_data != expected_data[i]: + print(f'❌ Payload {i + 1}: Data integrity mismatch') + passed = False + else: + print(f'✅ Payload {i + 1}: Data integrity verified') + elif expected_types[i] == 'table': + if len(decoded_data) > 0: + print(f'✅ Payload {i + 1}: Arrow IPC data present ({len(decoded_data)} bytes)') + else: + print(f'❌ Payload {i + 1}: Arrow IPC data is empty') + passed = False + + print(f' Size: {payload["size"]} bytes\n') + + # Test with chat-like payload (text + image + audio) + print('=== Chat-like Payload Test ===') + chat_data = [ + ('text', 'Hello!', 'text'), + ('image', bytes([0xFF, 0xD8, 0xFF, 0xE0]), 'image'), + ('audio', bytes([0x46, 0x4C, 0x41, 0x43]), 'audio') + ] + + chat_env, _ = await smartsend( + TEST_SUBJECT, + chat_data, + broker_url=TEST_BROKER_URL, + fileserver_url=TEST_FILESERVER_URL, + correlation_id='chat-' + correlation_id, + is_publish=False + ) + + if len(chat_env['payloads']) == 3: + print('✅ Chat-like payloads handled correctly') + else: + print('❌ Chat-like payloads handling failed') + passed = False + + # Final result + print('\n=== Test Result ===') + if passed: + print('✅ ALL TESTS PASSED') + sys.exit(0) + else: + print('❌ SOME TESTS FAILED') + sys.exit(1) + + except Exception as e: + print(f'❌ Test failed with error: {e}') + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == '__main__': + asyncio.run(run_test()) \ No newline at end of file diff --git a/testFile_large.zip b/testFile_large.zip deleted file mode 100644 index 56dfcd1..0000000 Binary files a/testFile_large.zip and /dev/null differ diff --git a/testFile_small.zip b/testFile_small.zip deleted file mode 100644 index 75f1413..0000000 Binary files a/testFile_small.zip and /dev/null differ