From 49d789872086260e29a73a0394c51024ee881d93 Mon Sep 17 00:00:00 2001 From: narawat Date: Sun, 15 Mar 2026 11:58:08 +0700 Subject: [PATCH] remove arrow support for natsbridge_csr.js --- README.md | 36 +++++++++++++++++----- docs/architecture.md | 14 +++++++-- docs/requirements.md | 17 ++++++----- docs/spec.md | 17 +++++++++-- src/natsbridge_csr.js | 71 +++---------------------------------------- 5 files changed, 69 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index d4f33a7..27a3e2d 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ NATSBridge enables seamless communication across multiple platforms through NATS | Platform | Implementation | Features | |----------|----------------|----------| | **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch | -| **JavaScript (Node.js)** | [`src/natsbridge.js`](src/natsbridge_ssr.js) | Node.js, async/await | -| **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await | -| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints | +| **JavaScript (Node.js)** | [`src/natsbridge_ssr.js`](src/natsbridge_ssr.js) | Node.js, async/await, Arrow IPC | +| **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await, JSON table only | +| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints, Arrow IPC | | **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API | ### Platform Comparison @@ -57,7 +57,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS | Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ | | Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) | | Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ | -| Arrow IPC | ✅ Native | ✅ | ✅ | ✅ | ❌ | +| Arrow IPC | ✅ Native | ✅ Native | ❌ (Browser incompatible) | ✅ Native | ❌ | +| JSON Table | ✅ | ✅ | ✅ (Only table type) | ✅ | ⚠️ (Limited) | | Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ | | Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) | | Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ | @@ -73,7 +74,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS - ✅ **Multi-payload support** - send multiple payloads with different types in one message - ✅ **Automatic transport selection** - direct vs link based on payload size - ✅ **Claim-Check pattern** for payloads ≥ 500KB -- ✅ **Apache Arrow IPC** support for tabular data (zero-copy reading) +- ✅ **Apache Arrow IPC** support for tabular data (Desktop: Julia/Python/Node.js) +- ✅ **JSON Table** support for tabular data (All platforms including Browser) - ✅ **Exponential backoff** for reliable file server downloads - ✅ **Correlation ID tracking** for message tracing - ✅ **Reply-to support** for request-response patterns @@ -424,8 +426,8 @@ env = NATSBridge.smartreceive( |------|-------|------------|--------|-------------|-------------| | `text` | `String` | `string` | `str` | `str` | Plain text strings | | `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries | -| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | -| `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array` | `list[dict]` | ⚠️ | Tabular data (JSON) | +| `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | +| `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array` | `list[dict]` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** | | `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) | | `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) | | `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) | @@ -611,6 +613,26 @@ data = [("students", df, "arrowtable")] env, env_json_str = await NATSBridge.smartsend("/data/analysis", data) ``` +#### JavaScript (Browser) + +```javascript +import NATSBridge from './src/natsbridge_csr.js'; + +// Browser uses jsontable (JSON array of objects) instead of arrowtable +// Apache Arrow is not compatible with browsers +const df = [ + { id: 1, name: "Alice", score: 95 }, + { id: 2, name: "Bob", score: 88 }, + { id: 3, name: "Charlie", score: 92 } +]; + +const [env, env_json_str] = await NATSBridge.smartsend( + "/data/analysis", + [["students", df, "jsontable"]], // Use jsontable for browser + { broker_url: 'ws://localhost:4222' } +); +``` + ### Example 4: Request-Response Pattern Bi-directional communication with reply-to support. diff --git a/docs/architecture.md b/docs/architecture.md index 340679b..3aff666 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -275,8 +275,8 @@ end |------|-------------|---------------|----------|-----------| | `text` | Plain text string | UTF-8 bytes | Base64 | All | | `dictionary` | JSON object | JSON string | Base64/JSON | All | -| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop | -| `jsontable` | JSON array of objects | JSON string | Base64/json | All | +| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop (Julia/Python/Node.js) | +| `jsontable` | JSON array of objects | JSON string | Base64/json | All (including Browser) | | `image` | Binary image data | Raw bytes | Base64 | All | | `audio` | Binary audio data | Raw bytes | Base64 | All | | `video` | Binary video data | Raw bytes | Base64 | All | @@ -442,6 +442,16 @@ class NATSBridge: self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL ``` +### Browser Architecture + +Browser JavaScript has specific constraints due to security and compatibility: + +- **Async/await**: Native async/await support +- **No Apache Arrow**: Arrow IPC not available in browsers +- **JSON table only**: Use "jsontable" for tabular data +- **WebSocket NATS**: Uses nats.ws for browser-compatible NATS connections +- **Fetch API**: HTTP file server communication via fetch + ### MicroPython Architecture MicroPython has significant constraints: diff --git a/docs/requirements.md b/docs/requirements.md index dc6c0e4..a0cfaea 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -115,8 +115,9 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless | Platform | Minimum Version | Notes | |----------|-----------------|-------| | Julia | 1.7+ | Arrow.jl required for arrowtable support | -| Node.js | 16+ | nats.js required | +| Node.js | 16+ | nats.js required, Arrow IPC supported | | Python | 3.8+ | pyarrow required for arrowtable support | +| Browser | Latest | No Arrow IPC (uses jsontable only) | | MicroPython | 1.19+ | Limited to direct transport | --- @@ -181,8 +182,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless |------|-------|------------|--------|-------------|-------------| | `text` | `String` | `string` | `str` | `str` | Plain text strings | | `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable data | -| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | -| `jsontable` | `Vector{NamedTuple}` | `Array` | `list[dict]` | ⚠️ | Tabular data (JSON) | +| `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | +| `jsontable` | `Vector{NamedTuple}` | `Array` | `list[dict]` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** | | `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image binary data | | `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio binary data | | `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video binary data | @@ -194,8 +195,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless |--------------|-----------------|-------| | `text` | UTF-8 → Base64 | Text must be String type | | `dictionary` | JSON → Base64 | JSON.jl for Julia | -| `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow | -| `jsontable` | JSON → Base64 | Human-readable format | +| `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow (Desktop only) | +| `jsontable` | JSON → Base64 | Human-readable format - **Browser uses this only** | | `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved | --- @@ -294,7 +295,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless | Test Scenario | Success Criteria | |-------------|-----------------| | Cross-platform text message | Julia ↔ JavaScript ↔ Python | -| Cross-platform tabular data | Arrow IPC round-trip | +| Cross-platform tabular data (Desktop) | Arrow IPC round-trip | +| Cross-platform tabular data (Browser) | JSON table round-trip | | Large file transfer | File server upload/download | | Multi-payload mixed content | All payload types in one message | @@ -356,6 +358,7 @@ function smartreceive( | Python | nats-py | Latest stable | | Python | aiohttp | Latest stable | | Python | pyarrow | Latest stable | +| Browser | nats.ws | Latest stable | ### Optional Dependencies @@ -399,7 +402,7 @@ function smartreceive( | Version | Supported Platforms | |---------|---------------------| -| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, MicroPython 1.19+ | +| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, Browser (latest), MicroPython 1.19+ | --- diff --git a/docs/spec.md b/docs/spec.md index 7908319..e18f560 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -176,6 +176,7 @@ await smartsend("/agent/v1/process", data) | All | `String` | `"text"` | | All | `Dict`/`Object` | `"dictionary"` | | Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` | +| Browser | `Array` of objects | `"jsontable"` (only table type) | | All | `Array` of objects | `"jsontable"` | | All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` | | Desktop | `Arrow.Table` | `"arrowtable"` | @@ -203,8 +204,8 @@ await smartsend("/agent/v1/process", data) |-------|-------------|---------------------|------------------| | `text` | Plain text string | All | `base64` | | `dictionary` | JSON object/dictionary | All | `base64`, `json` | -| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/JS/Python) | `base64`, `arrow-ipc` | -| `jsontable` | JSON array of objects | All | `base64`, `json` | +| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/Python/Node.js) | `base64`, `arrow-ipc` | +| `jsontable` | JSON array of objects | All (including Browser) | `base64`, `json` | | `image` | Binary image data | All | `base64` | | `audio` | Binary audio data | All | `base64` | | `video` | Binary video data | All | `base64` | @@ -613,7 +614,7 @@ function fileserver_download_handler( ## Platform-Specific Constraints -### Desktop (Julia/JS/Python) +### Desktop (Julia/Python/Node.js) | Feature | Status | Notes | |---------|--------|-------| @@ -623,6 +624,16 @@ function fileserver_download_handler( | File server download | ✅ Supported | HTTP/HTTPS | | Size threshold | 500KB | Configurable | +### Browser (JavaScript) + +| Feature | Status | Notes | +|---------|--------|-------| +| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible | +| JSON table | ✅ Supported | Only table type available in browser | +| File server upload | ✅ Supported | HTTP/HTTPS | +| File server download | ✅ Supported | HTTP/HTTPS | +| Size threshold | 500KB | Configurable | + ### MicroPython | Feature | Status | Notes | diff --git a/src/natsbridge_csr.js b/src/natsbridge_csr.js index b8026d3..3db7e1c 100644 --- a/src/natsbridge_csr.js +++ b/src/natsbridge_csr.js @@ -6,7 +6,9 @@ * using NATS as the message bus, with support for both direct payload transport and * URL-based transport for larger payloads. * - * Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" + * Supported payload types: "text", "dictionary", "jsontable", "image", "audio", "video", "binary" + * Note: Browser version does NOT support Apache Arrow IPC (arrowtable) due to browser compatibility constraints. + * Use "jsontable" for tabular data in browser applications. * * Browser-compatible version uses: * - nats.ws for WebSocket-based NATS connections @@ -21,7 +23,6 @@ import * as nats from 'nats.ws'; // Use native fetch available in browsers -import { tableFromArrays, tableToIPC } from 'apache-arrow/browser'; // ---------------------------------------------- Constants ---------------------------------------------- // @@ -111,13 +112,6 @@ async function serializeData(data, payloadType) { } else if (payloadType === 'dictionary') { const jsonStr = JSON.stringify(data); return new Uint8Array(new TextEncoder().encode(jsonStr)); - } else if (payloadType === 'arrowtable') { - // Convert array of objects to Arrow IPC format - if (!Array.isArray(data) || data.length === 0) { - throw new Error('Arrow table data must be a non-empty array of objects'); - } - - return serializeArrowTable(data); } else if (payloadType === 'jsontable') { // Serialize array of objects to JSON format if (!Array.isArray(data)) { @@ -154,49 +148,6 @@ async function serializeData(data, payloadType) { } } -/** - * Helper function to properly serialize table data to Arrow IPC - * @param {Array} data - Array of objects representing table rows - * @returns {Uint8Array} Arrow IPC formatted buffer - */ -function serializeArrowTable(data) { - if (!Array.isArray(data) || data.length === 0) { - throw new Error('Table data must be a non-empty array of objects'); - } - - logTrace('serializeArrowTable', `Serializing table with ${data.length} rows`); - - // Convert array of objects to a key-value format expected by tableFromArrays - const columns = {}; - const keys = Object.keys(data[0]); - for (const key of keys) { - columns[key] = data.map(row => row[key]); - } - - logTrace('serializeArrowTable', `Columns: ${Object.keys(columns).join(', ')}`); - - const table = tableFromArrays(columns); - - logTrace('serializeArrowTable', `Arrow table created with ${table.numRows} rows, ${table.numCols} cols`); - - // Convert to IPC format - const ipcBuffer = tableToIPC(table); - - logTrace('serializeArrowTable', `IPC buffer type: ${typeof ipcBuffer}, byteLength: ${ipcBuffer.byteLength}`); - - const resultBuffer = new Uint8Array(ipcBuffer); - logTrace('serializeArrowTable', `Result buffer: ${resultBuffer.length} bytes`); - - // Debug: Show first 20 bytes in hex - const hexPreview = []; - for (let i = 0; i < Math.min(20, resultBuffer.length); i++) { - hexPreview.push(resultBuffer[i].toString(16).padStart(2, '0')); - } - logTrace('serializeArrowTable', `First 20 bytes (hex): ${hexPreview.join(' ')}`); - - return resultBuffer; -} - /** * Deserialize bytes to data based on type * @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes @@ -210,7 +161,7 @@ async function deserializeData(data, payloadType, correlationId) { logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`); // Debug: Show first 20 bytes in hex for binary data - if (payloadType === 'arrowtable' || payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') { + if (payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') { const hexPreview = []; for (let i = 0; i < Math.min(20, buffer.length); i++) { hexPreview.push(buffer[i].toString(16).padStart(2, '0')); @@ -227,18 +178,6 @@ async function deserializeData(data, payloadType, correlationId) { const result = JSON.parse(jsonStr); logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`); return result; - } else if (payloadType === 'arrowtable') { - logTrace(correlationId, `deserializeData: Attempting Arrow table deserialization`); - - try { - // Try tableFromIPC (browser API) - const table = tableFromIPC(buffer); - logTrace(correlationId, `deserializeData: Arrow table from IPC - rows=${table.numRows}, cols=${table.numCols}`); - return table; - } catch (e) { - logTrace(correlationId, `deserializeData: tableFromIPC failed: ${e.message}`); - throw new Error(`Unable to deserialize Arrow table: ${e.message}`); - } } else if (payloadType === 'jsontable') { const jsonStr = new TextDecoder().decode(buffer); const result = JSON.parse(jsonStr); @@ -478,8 +417,6 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { let encoding = 'base64'; if (payloadType === 'jsontable') { encoding = 'json'; - } else if (payloadType === 'arrowtable') { - encoding = 'arrow-ipc'; } return {