remove_arrow_in_natsbridge_csr #11

Merged
ton merged 3 commits from remove_arrow_in_natsbridge_csr into main 2026-03-19 04:12:28 +00:00
5 changed files with 69 additions and 86 deletions
Showing only changes of commit 49d7898720 - Show all commits

View File

@@ -45,9 +45,9 @@ NATSBridge enables seamless communication across multiple platforms through NATS
| Platform | Implementation | Features | | Platform | Implementation | Features |
|----------|----------------|----------| |----------|----------------|----------|
| **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch | | **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch |
| **JavaScript (Node.js)** | [`src/natsbridge.js`](src/natsbridge_ssr.js) | Node.js, async/await | | **JavaScript (Node.js)** | [`src/natsbridge_ssr.js`](src/natsbridge_ssr.js) | Node.js, async/await, Arrow IPC |
| **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await | | **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await, JSON table only |
| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints | | **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints, Arrow IPC |
| **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API | | **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API |
### Platform Comparison ### Platform Comparison
@@ -57,7 +57,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS
| Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ | | Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ |
| Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) | | Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) |
| Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ | | Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ |
| Arrow IPC | ✅ Native | ✅ | ✅ | ✅ | ❌ | | Arrow IPC | ✅ Native | ✅ Native | ❌ (Browser incompatible) | ✅ Native | ❌ |
| JSON Table | ✅ | ✅ | ✅ (Only table type) | ✅ | ⚠️ (Limited) |
| Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ | | Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ |
| Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) | | Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) |
| Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ | | Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ |
@@ -73,7 +74,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS
-**Multi-payload support** - send multiple payloads with different types in one message -**Multi-payload support** - send multiple payloads with different types in one message
-**Automatic transport selection** - direct vs link based on payload size -**Automatic transport selection** - direct vs link based on payload size
-**Claim-Check pattern** for payloads ≥ 500KB -**Claim-Check pattern** for payloads ≥ 500KB
-**Apache Arrow IPC** support for tabular data (zero-copy reading) -**Apache Arrow IPC** support for tabular data (Desktop: Julia/Python/Node.js)
-**JSON Table** support for tabular data (All platforms including Browser)
-**Exponential backoff** for reliable file server downloads -**Exponential backoff** for reliable file server downloads
-**Correlation ID tracking** for message tracing -**Correlation ID tracking** for message tracing
-**Reply-to support** for request-response patterns -**Reply-to support** for request-response patterns
@@ -424,8 +426,8 @@ env = NATSBridge.smartreceive(
|------|-------|------------|--------|-------------|-------------| |------|-------|------------|--------|-------------|-------------|
| `text` | `String` | `string` | `str` | `str` | Plain text strings | | `text` | `String` | `string` | `str` | `str` | Plain text strings |
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries | | `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries |
| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array<Object>` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | | `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) |
| `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) | | `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** |
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) | | `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) |
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) | | `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) |
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) | | `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) |
@@ -611,6 +613,26 @@ data = [("students", df, "arrowtable")]
env, env_json_str = await NATSBridge.smartsend("/data/analysis", data) env, env_json_str = await NATSBridge.smartsend("/data/analysis", data)
``` ```
#### JavaScript (Browser)
```javascript
import NATSBridge from './src/natsbridge_csr.js';
// Browser uses jsontable (JSON array of objects) instead of arrowtable
// Apache Arrow is not compatible with browsers
const df = [
{ id: 1, name: "Alice", score: 95 },
{ id: 2, name: "Bob", score: 88 },
{ id: 3, name: "Charlie", score: 92 }
];
const [env, env_json_str] = await NATSBridge.smartsend(
"/data/analysis",
[["students", df, "jsontable"]], // Use jsontable for browser
{ broker_url: 'ws://localhost:4222' }
);
```
### Example 4: Request-Response Pattern ### Example 4: Request-Response Pattern
Bi-directional communication with reply-to support. Bi-directional communication with reply-to support.

View File

@@ -275,8 +275,8 @@ end
|------|-------------|---------------|----------|-----------| |------|-------------|---------------|----------|-----------|
| `text` | Plain text string | UTF-8 bytes | Base64 | All | | `text` | Plain text string | UTF-8 bytes | Base64 | All |
| `dictionary` | JSON object | JSON string | Base64/JSON | All | | `dictionary` | JSON object | JSON string | Base64/JSON | All |
| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop | | `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop (Julia/Python/Node.js) |
| `jsontable` | JSON array of objects | JSON string | Base64/json | All | | `jsontable` | JSON array of objects | JSON string | Base64/json | All (including Browser) |
| `image` | Binary image data | Raw bytes | Base64 | All | | `image` | Binary image data | Raw bytes | Base64 | All |
| `audio` | Binary audio data | Raw bytes | Base64 | All | | `audio` | Binary audio data | Raw bytes | Base64 | All |
| `video` | Binary video data | Raw bytes | Base64 | All | | `video` | Binary video data | Raw bytes | Base64 | All |
@@ -442,6 +442,16 @@ class NATSBridge:
self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL
``` ```
### Browser Architecture
Browser JavaScript has specific constraints due to security and compatibility:
- **Async/await**: Native async/await support
- **No Apache Arrow**: Arrow IPC not available in browsers
- **JSON table only**: Use "jsontable" for tabular data
- **WebSocket NATS**: Uses nats.ws for browser-compatible NATS connections
- **Fetch API**: HTTP file server communication via fetch
### MicroPython Architecture ### MicroPython Architecture
MicroPython has significant constraints: MicroPython has significant constraints:

View File

@@ -115,8 +115,9 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
| Platform | Minimum Version | Notes | | Platform | Minimum Version | Notes |
|----------|-----------------|-------| |----------|-----------------|-------|
| Julia | 1.7+ | Arrow.jl required for arrowtable support | | Julia | 1.7+ | Arrow.jl required for arrowtable support |
| Node.js | 16+ | nats.js required | | Node.js | 16+ | nats.js required, Arrow IPC supported |
| Python | 3.8+ | pyarrow required for arrowtable support | | Python | 3.8+ | pyarrow required for arrowtable support |
| Browser | Latest | No Arrow IPC (uses jsontable only) |
| MicroPython | 1.19+ | Limited to direct transport | | MicroPython | 1.19+ | Limited to direct transport |
--- ---
@@ -181,8 +182,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
|------|-------|------------|--------|-------------|-------------| |------|-------|------------|--------|-------------|-------------|
| `text` | `String` | `string` | `str` | `str` | Plain text strings | | `text` | `String` | `string` | `str` | `str` | Plain text strings |
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable data | | `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable data |
| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array<Object>` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | | `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) |
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) | | `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** |
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image binary data | | `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image binary data |
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio binary data | | `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio binary data |
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video binary data | | `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video binary data |
@@ -194,8 +195,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
|--------------|-----------------|-------| |--------------|-----------------|-------|
| `text` | UTF-8 → Base64 | Text must be String type | | `text` | UTF-8 → Base64 | Text must be String type |
| `dictionary` | JSON → Base64 | JSON.jl for Julia | | `dictionary` | JSON → Base64 | JSON.jl for Julia |
| `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow | | `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow (Desktop only) |
| `jsontable` | JSON → Base64 | Human-readable format | | `jsontable` | JSON → Base64 | Human-readable format - **Browser uses this only** |
| `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved | | `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved |
--- ---
@@ -294,7 +295,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
| Test Scenario | Success Criteria | | Test Scenario | Success Criteria |
|-------------|-----------------| |-------------|-----------------|
| Cross-platform text message | Julia ↔ JavaScript ↔ Python | | Cross-platform text message | Julia ↔ JavaScript ↔ Python |
| Cross-platform tabular data | Arrow IPC round-trip | | Cross-platform tabular data (Desktop) | Arrow IPC round-trip |
| Cross-platform tabular data (Browser) | JSON table round-trip |
| Large file transfer | File server upload/download | | Large file transfer | File server upload/download |
| Multi-payload mixed content | All payload types in one message | | Multi-payload mixed content | All payload types in one message |
@@ -356,6 +358,7 @@ function smartreceive(
| Python | nats-py | Latest stable | | Python | nats-py | Latest stable |
| Python | aiohttp | Latest stable | | Python | aiohttp | Latest stable |
| Python | pyarrow | Latest stable | | Python | pyarrow | Latest stable |
| Browser | nats.ws | Latest stable |
### Optional Dependencies ### Optional Dependencies
@@ -399,7 +402,7 @@ function smartreceive(
| Version | Supported Platforms | | Version | Supported Platforms |
|---------|---------------------| |---------|---------------------|
| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, MicroPython 1.19+ | | v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, Browser (latest), MicroPython 1.19+ |
--- ---

View File

@@ -176,6 +176,7 @@ await smartsend("/agent/v1/process", data)
| All | `String` | `"text"` | | All | `String` | `"text"` |
| All | `Dict`/`Object` | `"dictionary"` | | All | `Dict`/`Object` | `"dictionary"` |
| Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` | | Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` |
| Browser | `Array` of objects | `"jsontable"` (only table type) |
| All | `Array` of objects | `"jsontable"` | | All | `Array` of objects | `"jsontable"` |
| All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` | | All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` |
| Desktop | `Arrow.Table` | `"arrowtable"` | | Desktop | `Arrow.Table` | `"arrowtable"` |
@@ -203,8 +204,8 @@ await smartsend("/agent/v1/process", data)
|-------|-------------|---------------------|------------------| |-------|-------------|---------------------|------------------|
| `text` | Plain text string | All | `base64` | | `text` | Plain text string | All | `base64` |
| `dictionary` | JSON object/dictionary | All | `base64`, `json` | | `dictionary` | JSON object/dictionary | All | `base64`, `json` |
| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/JS/Python) | `base64`, `arrow-ipc` | | `arrowtable` | Apache Arrow IPC table | Desktop (Julia/Python/Node.js) | `base64`, `arrow-ipc` |
| `jsontable` | JSON array of objects | All | `base64`, `json` | | `jsontable` | JSON array of objects | All (including Browser) | `base64`, `json` |
| `image` | Binary image data | All | `base64` | | `image` | Binary image data | All | `base64` |
| `audio` | Binary audio data | All | `base64` | | `audio` | Binary audio data | All | `base64` |
| `video` | Binary video data | All | `base64` | | `video` | Binary video data | All | `base64` |
@@ -613,7 +614,7 @@ function fileserver_download_handler(
## Platform-Specific Constraints ## Platform-Specific Constraints
### Desktop (Julia/JS/Python) ### Desktop (Julia/Python/Node.js)
| Feature | Status | Notes | | Feature | Status | Notes |
|---------|--------|-------| |---------|--------|-------|
@@ -623,6 +624,16 @@ function fileserver_download_handler(
| File server download | ✅ Supported | HTTP/HTTPS | | File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable | | Size threshold | 500KB | Configurable |
### Browser (JavaScript)
| Feature | Status | Notes |
|---------|--------|-------|
| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible |
| JSON table | ✅ Supported | Only table type available in browser |
| File server upload | ✅ Supported | HTTP/HTTPS |
| File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable |
### MicroPython ### MicroPython
| Feature | Status | Notes | | Feature | Status | Notes |

View File

@@ -6,7 +6,9 @@
* using NATS as the message bus, with support for both direct payload transport and * using NATS as the message bus, with support for both direct payload transport and
* URL-based transport for larger payloads. * URL-based transport for larger payloads.
* *
* Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * Supported payload types: "text", "dictionary", "jsontable", "image", "audio", "video", "binary"
* Note: Browser version does NOT support Apache Arrow IPC (arrowtable) due to browser compatibility constraints.
* Use "jsontable" for tabular data in browser applications.
* *
* Browser-compatible version uses: * Browser-compatible version uses:
* - nats.ws for WebSocket-based NATS connections * - nats.ws for WebSocket-based NATS connections
@@ -21,7 +23,6 @@
import * as nats from 'nats.ws'; import * as nats from 'nats.ws';
// Use native fetch available in browsers // Use native fetch available in browsers
import { tableFromArrays, tableToIPC } from 'apache-arrow/browser';
// ---------------------------------------------- Constants ---------------------------------------------- // // ---------------------------------------------- Constants ---------------------------------------------- //
@@ -111,13 +112,6 @@ async function serializeData(data, payloadType) {
} else if (payloadType === 'dictionary') { } else if (payloadType === 'dictionary') {
const jsonStr = JSON.stringify(data); const jsonStr = JSON.stringify(data);
return new Uint8Array(new TextEncoder().encode(jsonStr)); return new Uint8Array(new TextEncoder().encode(jsonStr));
} else if (payloadType === 'arrowtable') {
// Convert array of objects to Arrow IPC format
if (!Array.isArray(data) || data.length === 0) {
throw new Error('Arrow table data must be a non-empty array of objects');
}
return serializeArrowTable(data);
} else if (payloadType === 'jsontable') { } else if (payloadType === 'jsontable') {
// Serialize array of objects to JSON format // Serialize array of objects to JSON format
if (!Array.isArray(data)) { if (!Array.isArray(data)) {
@@ -154,49 +148,6 @@ async function serializeData(data, payloadType) {
} }
} }
/**
* Helper function to properly serialize table data to Arrow IPC
* @param {Array<Object>} data - Array of objects representing table rows
* @returns {Uint8Array} Arrow IPC formatted buffer
*/
function serializeArrowTable(data) {
if (!Array.isArray(data) || data.length === 0) {
throw new Error('Table data must be a non-empty array of objects');
}
logTrace('serializeArrowTable', `Serializing table with ${data.length} rows`);
// Convert array of objects to a key-value format expected by tableFromArrays
const columns = {};
const keys = Object.keys(data[0]);
for (const key of keys) {
columns[key] = data.map(row => row[key]);
}
logTrace('serializeArrowTable', `Columns: ${Object.keys(columns).join(', ')}`);
const table = tableFromArrays(columns);
logTrace('serializeArrowTable', `Arrow table created with ${table.numRows} rows, ${table.numCols} cols`);
// Convert to IPC format
const ipcBuffer = tableToIPC(table);
logTrace('serializeArrowTable', `IPC buffer type: ${typeof ipcBuffer}, byteLength: ${ipcBuffer.byteLength}`);
const resultBuffer = new Uint8Array(ipcBuffer);
logTrace('serializeArrowTable', `Result buffer: ${resultBuffer.length} bytes`);
// Debug: Show first 20 bytes in hex
const hexPreview = [];
for (let i = 0; i < Math.min(20, resultBuffer.length); i++) {
hexPreview.push(resultBuffer[i].toString(16).padStart(2, '0'));
}
logTrace('serializeArrowTable', `First 20 bytes (hex): ${hexPreview.join(' ')}`);
return resultBuffer;
}
/** /**
* Deserialize bytes to data based on type * Deserialize bytes to data based on type
* @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes * @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes
@@ -210,7 +161,7 @@ async function deserializeData(data, payloadType, correlationId) {
logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`); logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`);
// Debug: Show first 20 bytes in hex for binary data // Debug: Show first 20 bytes in hex for binary data
if (payloadType === 'arrowtable' || payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') { if (payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') {
const hexPreview = []; const hexPreview = [];
for (let i = 0; i < Math.min(20, buffer.length); i++) { for (let i = 0; i < Math.min(20, buffer.length); i++) {
hexPreview.push(buffer[i].toString(16).padStart(2, '0')); hexPreview.push(buffer[i].toString(16).padStart(2, '0'));
@@ -227,18 +178,6 @@ async function deserializeData(data, payloadType, correlationId) {
const result = JSON.parse(jsonStr); const result = JSON.parse(jsonStr);
logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`); logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`);
return result; return result;
} else if (payloadType === 'arrowtable') {
logTrace(correlationId, `deserializeData: Attempting Arrow table deserialization`);
try {
// Try tableFromIPC (browser API)
const table = tableFromIPC(buffer);
logTrace(correlationId, `deserializeData: Arrow table from IPC - rows=${table.numRows}, cols=${table.numCols}`);
return table;
} catch (e) {
logTrace(correlationId, `deserializeData: tableFromIPC failed: ${e.message}`);
throw new Error(`Unable to deserialize Arrow table: ${e.message}`);
}
} else if (payloadType === 'jsontable') { } else if (payloadType === 'jsontable') {
const jsonStr = new TextDecoder().decode(buffer); const jsonStr = new TextDecoder().decode(buffer);
const result = JSON.parse(jsonStr); const result = JSON.parse(jsonStr);
@@ -478,8 +417,6 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) {
let encoding = 'base64'; let encoding = 'base64';
if (payloadType === 'jsontable') { if (payloadType === 'jsontable') {
encoding = 'json'; encoding = 'json';
} else if (payloadType === 'arrowtable') {
encoding = 'arrow-ipc';
} }
return { return {