From efd77937a2598ec8ad756de95b4d9ec0b7d739a7 Mon Sep 17 00:00:00 2001 From: narawat Date: Sat, 23 May 2026 14:29:10 +0700 Subject: [PATCH] update readme --- README.md | 95 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 176a99c..0b8a090 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ -# msghandler - Cross-Platform Bi-Directional Data Bridge +# msghandler - Cross-Platform Communication Layer -A high-performance, bi-directional data bridge for **Julia**, **JavaScript**, **Python**, and **MicroPython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +A high-performance, **transport-agnostic** communication layer for **Julia**, **JavaScript**, **Python**, and **MicroPython** applications. Implements the Claim-Check pattern for efficient payload transport (direct for small payloads, URL-based for large payloads). [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) -[![NATS](https://img.shields.io/badge/NATS-Enabled-green.svg)](https://nats.io) --- @@ -90,12 +89,12 @@ const [env, json_str] = await smartpack("subject", data, options); ## Overview -msghandler enables seamless communication across multiple platforms through NATS, with intelligent transport selection based on payload size: +msghandler provides a **transport-agnostic** communication layer with intelligent payload transport selection: | Transport | Payload Size | Method | |-----------|--------------|--------| -| **Direct** | < 500KB | Sent directly via NATS (Base64 encoded) | -| **Link** | ≥ 500KB | Uploaded to HTTP file server, URL sent via NATS | +| **Direct** | < 500KB | Sent directly via chosen transport (Base64 encoded) | +| **Link** | ≥ 500KB | Uploaded to HTTP file server, URL sent via chosen transport | ### Use Cases @@ -103,6 +102,14 @@ msghandler enables seamless communication across multiple platforms through NATS - **File Transfer**: Efficient transfer of large files using claim-check pattern - **IoT/Embedded**: Sensor data, telemetry, and analytics pipelines (MicroPython) - **Cross-Platform Communication**: Interoperability between Julia, JavaScript, Python, and MicroPython systems +- **Any Transport**: Works with NATS, HTTP, WebSockets, WebRTC, or any custom transport mechanism + +### Key Design Principles + +- **Transport Agnostic**: Core API (`smartpack`/`smartunpack`) is decoupled from transport +- **Claim-Check Pattern**: Efficient handling of large payloads via URL references +- **Type System**: Rich payload types (text, dictionary, arrowtable, jsontable, binary) +- **Cross-Platform**: Unified API across Julia, JavaScript, Python, and MicroPython --- @@ -112,7 +119,7 @@ msghandler enables seamless communication across multiple platforms through NATS |----------|----------------|----------| | **Julia** | [`src/msghandler.jl`](src/msghandler.jl) | Full feature set, Arrow IPC, multiple dispatch | | **JavaScript (Node.js)** | [`src/msghandler_ssr.js`](src/msghandler_ssr.js) | Node.js, async/await, Arrow IPC | -| **JavaScript (Browser)** | [`src/msghandler_csr.js`](src/msghandler_csr.js) | Browser, WebSocket NATS, async/await, JSON table only | +| **JavaScript (Browser)** | [`src/msghandler_csr.js`](src/msghandler_csr.js) | Browser, async/await, JSON table only | | **Python** | [`src/msghandler.py`](src/msghandler.py) | Desktop Python, asyncio, type hints, Arrow IPC | | **MicroPython** | [`src/msghandler_mpy.py`](src/msghandler_mpy.py) | Memory-constrained, synchronous API | @@ -129,23 +136,21 @@ msghandler enables seamless communication across multiple platforms through NATS | Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) | | Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ | | Cross-Platform API | ✅ | ✅ | ✅ | ✅ | ✅ | -| WebSocket NATS | ❌ | ❌ | ✅ | ❌ | ❌ | --- ## Features -- ✅ **Cross-platform messaging** for Julia, JavaScript, Python, and MicroPython applications -- ✅ **Bi-directional messaging** with request-reply patterns -- ✅ **Multi-payload support** - send multiple payloads with different types in one message -- ✅ **Automatic transport selection** - direct vs link based on payload size -- ✅ **Claim-Check pattern** for payloads ≥ 500KB -- ✅ **Apache Arrow IPC** support for tabular data (Desktop: Julia/Python/Node.js) -- ✅ **JSON Table** support for tabular data (All platforms including Browser) -- ✅ **Exponential backoff** for reliable file server downloads -- ✅ **Correlation ID tracking** for message tracing -- ✅ **Reply-to support** for request-response patterns -- ✅ **Handler function abstraction** - pluggable file server implementations (Plik, AWS S3, custom) +- ✅ **Transport-agnostic** - Core API works with any communication channel (NATS, HTTP, WebSockets, etc.) +- ✅ **Cross-platform** - Unified API for Julia, JavaScript, Python, and MicroPython +- ✅ **Bi-directional** - Request-reply patterns with `reply_to` support +- ✅ **Multi-payload** - Send multiple payloads of different types in one message +- ✅ **Automatic transport selection** - Direct vs link based on payload size (threshold: 500KB) +- ✅ **Claim-Check pattern** - Efficient large payload handling via URL references +- ✅ **Rich payload types** - Text, dictionary, Arrow IPC, JSON table, image, audio, video, binary +- ✅ **Exponential backoff** - Reliable file server downloads with retry logic +- ✅ **Correlation ID** - End-to-end message tracing +- ✅ **Handler abstraction** - Pluggable file server implementations (Plik, AWS S3, custom) --- @@ -153,20 +158,20 @@ msghandler enables seamless communication across multiple platforms through NATS ### Prerequisites -1. **NATS Server** - Install and run a NATS server: - ```bash - docker run -p 4222:4222 nats:latest - ``` +1. **Transport Channel** - Choose your communication channel: + - **NATS** (recommended): `docker run -p 4222:4222 nats:latest` + - **HTTP/WebSocket**: Any HTTP server or WebSocket endpoint + - **Custom**: Implement your own transport mechanism -2. **HTTP File Server** (optional, for large payloads) - Install and run a file server: - ```bash - # Using Plik - docker run -p 8080:8080 -v /tmp/fileserver:/var/lib/plik -e PLIK_ADMIN_PASSWORD=admin plik/plik - - # OR using simple Python HTTP server - mkdir -p /tmp/fileserver - python3 -m http.server 8080 --directory /tmp/fileserver - ``` +2. **HTTP File Server** (optional, for payloads ≥ 500KB) - Install and run: + ```bash + # Using Plik + docker run -p 8080:8080 -v /tmp/fileserver:/var/lib/plik -e PLIK_ADMIN_PASSWORD=admin plik/plik + + # OR using Python HTTP server + mkdir -p /tmp/fileserver + python3 -m http.server 8080 --directory /tmp/fileserver + ``` ### Send Your First Message @@ -191,12 +196,16 @@ filename_large_image = basename(file_path_large_image) payload_2 = (filename_large_image, file_data_large_image, "binary") payloads = [payload_1, payload_2] # List of tuples + +# Step 1: Create the message envelope (transport-agnostic) envelope, envelope_json_str = msghandler.smartpack("test.topic", payloads; broker_url="nats.yiem.cc", fileserver_url="http://192.168.88.104:8080") -conn = NATS.connect("nats.yiem.cc") # Create NATS connection -NATS.publish(conn, "test.topic", envelope_json_str) # Publish message to NATS + +# Step 2: Send via your chosen transport (NATS in this example) +conn = NATS.connect("nats.yiem.cc") +NATS.publish(conn, "test.topic", envelope_json_str) NATS.drain(conn) ``` @@ -244,9 +253,13 @@ data = [("message", "Hello World", "text")] env, env_json_str = await smartpack( "/chat/room1", data, - broker_url="nats://localhost:4222" + broker_url="nats://localhost:4222", + fileserver_url="http://localhost:8080" ) -print("Message sent!") +print("Envelope created:", env) + +# To send via transport (e.g., NATS, HTTP, WebSocket): +# transport_send_function("/chat/room1", env_json_str) ``` #### MicroPython @@ -259,9 +272,13 @@ env, env_json_str = smartpack( "/chat/room1", data, broker_url="nats://localhost:4222", + fileserver_url="http://localhost:8080", size_threshold=100000 ) -print("Message sent!") +print("Envelope created:", env) + +# To send via transport (e.g., HTTP POST, WebSocket): +# transport_send_function("/chat/room1", env_json_str) ``` ### Receive Your First Message @@ -271,11 +288,13 @@ print("Message sent!") ```julia using msghandler, NATS +# Step 1: Receive message from your transport (NATS in this example) conn = NATS.connect("nats.yiem.cc") NATS.subscribe(conn, "test.topic") do msg println("Received message on $(msg.subject)") envelope_json_str = String(msg.payload) + # Step 2: Unpack the envelope (transport-agnostic) envelope = msghandler.smartunpack( envelope_json_str; max_retries = 5, @@ -384,7 +403,7 @@ const data = [ ### smartpack -Sends data either directly via NATS or via a fileserver URL, depending on payload size. +Sends data via your chosen transport mechanism (NATS, HTTP, WebSocket, etc.) with intelligent transport selection (direct vs URL-based) based on payload size. #### Julia