remove_arrow_in_natsbridge_csr #11
36
README.md
36
README.md
@@ -45,9 +45,9 @@ NATSBridge enables seamless communication across multiple platforms through NATS
|
|||||||
| Platform | Implementation | Features |
|
| Platform | Implementation | Features |
|
||||||
|----------|----------------|----------|
|
|----------|----------------|----------|
|
||||||
| **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch |
|
| **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch |
|
||||||
| **JavaScript (Node.js)** | [`src/natsbridge.js`](src/natsbridge_ssr.js) | Node.js, async/await |
|
| **JavaScript (Node.js)** | [`src/natsbridge_ssr.js`](src/natsbridge_ssr.js) | Node.js, async/await, Arrow IPC |
|
||||||
| **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await |
|
| **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await, JSON table only |
|
||||||
| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints |
|
| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints, Arrow IPC |
|
||||||
| **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API |
|
| **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API |
|
||||||
|
|
||||||
### Platform Comparison
|
### Platform Comparison
|
||||||
@@ -57,7 +57,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS
|
|||||||
| Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ |
|
| Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ |
|
||||||
| Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) |
|
| Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) |
|
||||||
| Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ |
|
| Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ |
|
||||||
| Arrow IPC | ✅ Native | ✅ | ✅ | ✅ | ❌ |
|
| Arrow IPC | ✅ Native | ✅ Native | ❌ (Browser incompatible) | ✅ Native | ❌ |
|
||||||
|
| JSON Table | ✅ | ✅ | ✅ (Only table type) | ✅ | ⚠️ (Limited) |
|
||||||
| Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||||
| Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) |
|
| Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) |
|
||||||
| Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||||
@@ -73,7 +74,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS
|
|||||||
- ✅ **Multi-payload support** - send multiple payloads with different types in one message
|
- ✅ **Multi-payload support** - send multiple payloads with different types in one message
|
||||||
- ✅ **Automatic transport selection** - direct vs link based on payload size
|
- ✅ **Automatic transport selection** - direct vs link based on payload size
|
||||||
- ✅ **Claim-Check pattern** for payloads ≥ 500KB
|
- ✅ **Claim-Check pattern** for payloads ≥ 500KB
|
||||||
- ✅ **Apache Arrow IPC** support for tabular data (zero-copy reading)
|
- ✅ **Apache Arrow IPC** support for tabular data (Desktop: Julia/Python/Node.js)
|
||||||
|
- ✅ **JSON Table** support for tabular data (All platforms including Browser)
|
||||||
- ✅ **Exponential backoff** for reliable file server downloads
|
- ✅ **Exponential backoff** for reliable file server downloads
|
||||||
- ✅ **Correlation ID tracking** for message tracing
|
- ✅ **Correlation ID tracking** for message tracing
|
||||||
- ✅ **Reply-to support** for request-response patterns
|
- ✅ **Reply-to support** for request-response patterns
|
||||||
@@ -424,8 +426,8 @@ env = NATSBridge.smartreceive(
|
|||||||
|------|-------|------------|--------|-------------|-------------|
|
|------|-------|------------|--------|-------------|-------------|
|
||||||
| `text` | `String` | `string` | `str` | `str` | Plain text strings |
|
| `text` | `String` | `string` | `str` | `str` | Plain text strings |
|
||||||
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries |
|
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries |
|
||||||
| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array<Object>` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) |
|
| `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) |
|
||||||
| `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) |
|
| `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** |
|
||||||
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) |
|
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) |
|
||||||
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) |
|
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) |
|
||||||
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) |
|
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) |
|
||||||
@@ -611,6 +613,26 @@ data = [("students", df, "arrowtable")]
|
|||||||
env, env_json_str = await NATSBridge.smartsend("/data/analysis", data)
|
env, env_json_str = await NATSBridge.smartsend("/data/analysis", data)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### JavaScript (Browser)
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import NATSBridge from './src/natsbridge_csr.js';
|
||||||
|
|
||||||
|
// Browser uses jsontable (JSON array of objects) instead of arrowtable
|
||||||
|
// Apache Arrow is not compatible with browsers
|
||||||
|
const df = [
|
||||||
|
{ id: 1, name: "Alice", score: 95 },
|
||||||
|
{ id: 2, name: "Bob", score: 88 },
|
||||||
|
{ id: 3, name: "Charlie", score: 92 }
|
||||||
|
];
|
||||||
|
|
||||||
|
const [env, env_json_str] = await NATSBridge.smartsend(
|
||||||
|
"/data/analysis",
|
||||||
|
[["students", df, "jsontable"]], // Use jsontable for browser
|
||||||
|
{ broker_url: 'ws://localhost:4222' }
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
### Example 4: Request-Response Pattern
|
### Example 4: Request-Response Pattern
|
||||||
|
|
||||||
Bi-directional communication with reply-to support.
|
Bi-directional communication with reply-to support.
|
||||||
|
|||||||
@@ -275,8 +275,8 @@ end
|
|||||||
|------|-------------|---------------|----------|-----------|
|
|------|-------------|---------------|----------|-----------|
|
||||||
| `text` | Plain text string | UTF-8 bytes | Base64 | All |
|
| `text` | Plain text string | UTF-8 bytes | Base64 | All |
|
||||||
| `dictionary` | JSON object | JSON string | Base64/JSON | All |
|
| `dictionary` | JSON object | JSON string | Base64/JSON | All |
|
||||||
| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop |
|
| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop (Julia/Python/Node.js) |
|
||||||
| `jsontable` | JSON array of objects | JSON string | Base64/json | All |
|
| `jsontable` | JSON array of objects | JSON string | Base64/json | All (including Browser) |
|
||||||
| `image` | Binary image data | Raw bytes | Base64 | All |
|
| `image` | Binary image data | Raw bytes | Base64 | All |
|
||||||
| `audio` | Binary audio data | Raw bytes | Base64 | All |
|
| `audio` | Binary audio data | Raw bytes | Base64 | All |
|
||||||
| `video` | Binary video data | Raw bytes | Base64 | All |
|
| `video` | Binary video data | Raw bytes | Base64 | All |
|
||||||
@@ -442,6 +442,16 @@ class NATSBridge:
|
|||||||
self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL
|
self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Browser Architecture
|
||||||
|
|
||||||
|
Browser JavaScript has specific constraints due to security and compatibility:
|
||||||
|
|
||||||
|
- **Async/await**: Native async/await support
|
||||||
|
- **No Apache Arrow**: Arrow IPC not available in browsers
|
||||||
|
- **JSON table only**: Use "jsontable" for tabular data
|
||||||
|
- **WebSocket NATS**: Uses nats.ws for browser-compatible NATS connections
|
||||||
|
- **Fetch API**: HTTP file server communication via fetch
|
||||||
|
|
||||||
### MicroPython Architecture
|
### MicroPython Architecture
|
||||||
|
|
||||||
MicroPython has significant constraints:
|
MicroPython has significant constraints:
|
||||||
|
|||||||
@@ -115,8 +115,9 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
|
|||||||
| Platform | Minimum Version | Notes |
|
| Platform | Minimum Version | Notes |
|
||||||
|----------|-----------------|-------|
|
|----------|-----------------|-------|
|
||||||
| Julia | 1.7+ | Arrow.jl required for arrowtable support |
|
| Julia | 1.7+ | Arrow.jl required for arrowtable support |
|
||||||
| Node.js | 16+ | nats.js required |
|
| Node.js | 16+ | nats.js required, Arrow IPC supported |
|
||||||
| Python | 3.8+ | pyarrow required for arrowtable support |
|
| Python | 3.8+ | pyarrow required for arrowtable support |
|
||||||
|
| Browser | Latest | No Arrow IPC (uses jsontable only) |
|
||||||
| MicroPython | 1.19+ | Limited to direct transport |
|
| MicroPython | 1.19+ | Limited to direct transport |
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -181,8 +182,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
|
|||||||
|------|-------|------------|--------|-------------|-------------|
|
|------|-------|------------|--------|-------------|-------------|
|
||||||
| `text` | `String` | `string` | `str` | `str` | Plain text strings |
|
| `text` | `String` | `string` | `str` | `str` | Plain text strings |
|
||||||
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable data |
|
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable data |
|
||||||
| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array<Object>` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) |
|
| `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) |
|
||||||
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) |
|
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** |
|
||||||
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image binary data |
|
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image binary data |
|
||||||
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio binary data |
|
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio binary data |
|
||||||
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video binary data |
|
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video binary data |
|
||||||
@@ -194,8 +195,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
|
|||||||
|--------------|-----------------|-------|
|
|--------------|-----------------|-------|
|
||||||
| `text` | UTF-8 → Base64 | Text must be String type |
|
| `text` | UTF-8 → Base64 | Text must be String type |
|
||||||
| `dictionary` | JSON → Base64 | JSON.jl for Julia |
|
| `dictionary` | JSON → Base64 | JSON.jl for Julia |
|
||||||
| `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow |
|
| `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow (Desktop only) |
|
||||||
| `jsontable` | JSON → Base64 | Human-readable format |
|
| `jsontable` | JSON → Base64 | Human-readable format - **Browser uses this only** |
|
||||||
| `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved |
|
| `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved |
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -294,7 +295,8 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
|
|||||||
| Test Scenario | Success Criteria |
|
| Test Scenario | Success Criteria |
|
||||||
|-------------|-----------------|
|
|-------------|-----------------|
|
||||||
| Cross-platform text message | Julia ↔ JavaScript ↔ Python |
|
| Cross-platform text message | Julia ↔ JavaScript ↔ Python |
|
||||||
| Cross-platform tabular data | Arrow IPC round-trip |
|
| Cross-platform tabular data (Desktop) | Arrow IPC round-trip |
|
||||||
|
| Cross-platform tabular data (Browser) | JSON table round-trip |
|
||||||
| Large file transfer | File server upload/download |
|
| Large file transfer | File server upload/download |
|
||||||
| Multi-payload mixed content | All payload types in one message |
|
| Multi-payload mixed content | All payload types in one message |
|
||||||
|
|
||||||
@@ -356,6 +358,7 @@ function smartreceive(
|
|||||||
| Python | nats-py | Latest stable |
|
| Python | nats-py | Latest stable |
|
||||||
| Python | aiohttp | Latest stable |
|
| Python | aiohttp | Latest stable |
|
||||||
| Python | pyarrow | Latest stable |
|
| Python | pyarrow | Latest stable |
|
||||||
|
| Browser | nats.ws | Latest stable |
|
||||||
|
|
||||||
### Optional Dependencies
|
### Optional Dependencies
|
||||||
|
|
||||||
@@ -399,7 +402,7 @@ function smartreceive(
|
|||||||
|
|
||||||
| Version | Supported Platforms |
|
| Version | Supported Platforms |
|
||||||
|---------|---------------------|
|
|---------|---------------------|
|
||||||
| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, MicroPython 1.19+ |
|
| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, Browser (latest), MicroPython 1.19+ |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
17
docs/spec.md
17
docs/spec.md
@@ -176,6 +176,7 @@ await smartsend("/agent/v1/process", data)
|
|||||||
| All | `String` | `"text"` |
|
| All | `String` | `"text"` |
|
||||||
| All | `Dict`/`Object` | `"dictionary"` |
|
| All | `Dict`/`Object` | `"dictionary"` |
|
||||||
| Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` |
|
| Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` |
|
||||||
|
| Browser | `Array` of objects | `"jsontable"` (only table type) |
|
||||||
| All | `Array` of objects | `"jsontable"` |
|
| All | `Array` of objects | `"jsontable"` |
|
||||||
| All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` |
|
| All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` |
|
||||||
| Desktop | `Arrow.Table` | `"arrowtable"` |
|
| Desktop | `Arrow.Table` | `"arrowtable"` |
|
||||||
@@ -203,8 +204,8 @@ await smartsend("/agent/v1/process", data)
|
|||||||
|-------|-------------|---------------------|------------------|
|
|-------|-------------|---------------------|------------------|
|
||||||
| `text` | Plain text string | All | `base64` |
|
| `text` | Plain text string | All | `base64` |
|
||||||
| `dictionary` | JSON object/dictionary | All | `base64`, `json` |
|
| `dictionary` | JSON object/dictionary | All | `base64`, `json` |
|
||||||
| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/JS/Python) | `base64`, `arrow-ipc` |
|
| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/Python/Node.js) | `base64`, `arrow-ipc` |
|
||||||
| `jsontable` | JSON array of objects | All | `base64`, `json` |
|
| `jsontable` | JSON array of objects | All (including Browser) | `base64`, `json` |
|
||||||
| `image` | Binary image data | All | `base64` |
|
| `image` | Binary image data | All | `base64` |
|
||||||
| `audio` | Binary audio data | All | `base64` |
|
| `audio` | Binary audio data | All | `base64` |
|
||||||
| `video` | Binary video data | All | `base64` |
|
| `video` | Binary video data | All | `base64` |
|
||||||
@@ -613,7 +614,7 @@ function fileserver_download_handler(
|
|||||||
|
|
||||||
## Platform-Specific Constraints
|
## Platform-Specific Constraints
|
||||||
|
|
||||||
### Desktop (Julia/JS/Python)
|
### Desktop (Julia/Python/Node.js)
|
||||||
|
|
||||||
| Feature | Status | Notes |
|
| Feature | Status | Notes |
|
||||||
|---------|--------|-------|
|
|---------|--------|-------|
|
||||||
@@ -623,6 +624,16 @@ function fileserver_download_handler(
|
|||||||
| File server download | ✅ Supported | HTTP/HTTPS |
|
| File server download | ✅ Supported | HTTP/HTTPS |
|
||||||
| Size threshold | 500KB | Configurable |
|
| Size threshold | 500KB | Configurable |
|
||||||
|
|
||||||
|
### Browser (JavaScript)
|
||||||
|
|
||||||
|
| Feature | Status | Notes |
|
||||||
|
|---------|--------|-------|
|
||||||
|
| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible |
|
||||||
|
| JSON table | ✅ Supported | Only table type available in browser |
|
||||||
|
| File server upload | ✅ Supported | HTTP/HTTPS |
|
||||||
|
| File server download | ✅ Supported | HTTP/HTTPS |
|
||||||
|
| Size threshold | 500KB | Configurable |
|
||||||
|
|
||||||
### MicroPython
|
### MicroPython
|
||||||
|
|
||||||
| Feature | Status | Notes |
|
| Feature | Status | Notes |
|
||||||
|
|||||||
@@ -6,7 +6,9 @@
|
|||||||
* using NATS as the message bus, with support for both direct payload transport and
|
* using NATS as the message bus, with support for both direct payload transport and
|
||||||
* URL-based transport for larger payloads.
|
* URL-based transport for larger payloads.
|
||||||
*
|
*
|
||||||
* Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary"
|
* Supported payload types: "text", "dictionary", "jsontable", "image", "audio", "video", "binary"
|
||||||
|
* Note: Browser version does NOT support Apache Arrow IPC (arrowtable) due to browser compatibility constraints.
|
||||||
|
* Use "jsontable" for tabular data in browser applications.
|
||||||
*
|
*
|
||||||
* Browser-compatible version uses:
|
* Browser-compatible version uses:
|
||||||
* - nats.ws for WebSocket-based NATS connections
|
* - nats.ws for WebSocket-based NATS connections
|
||||||
@@ -21,7 +23,6 @@
|
|||||||
import * as nats from 'nats.ws';
|
import * as nats from 'nats.ws';
|
||||||
|
|
||||||
// Use native fetch available in browsers
|
// Use native fetch available in browsers
|
||||||
import { tableFromArrays, tableToIPC } from 'apache-arrow/browser';
|
|
||||||
|
|
||||||
// ---------------------------------------------- Constants ---------------------------------------------- //
|
// ---------------------------------------------- Constants ---------------------------------------------- //
|
||||||
|
|
||||||
@@ -111,13 +112,6 @@ async function serializeData(data, payloadType) {
|
|||||||
} else if (payloadType === 'dictionary') {
|
} else if (payloadType === 'dictionary') {
|
||||||
const jsonStr = JSON.stringify(data);
|
const jsonStr = JSON.stringify(data);
|
||||||
return new Uint8Array(new TextEncoder().encode(jsonStr));
|
return new Uint8Array(new TextEncoder().encode(jsonStr));
|
||||||
} else if (payloadType === 'arrowtable') {
|
|
||||||
// Convert array of objects to Arrow IPC format
|
|
||||||
if (!Array.isArray(data) || data.length === 0) {
|
|
||||||
throw new Error('Arrow table data must be a non-empty array of objects');
|
|
||||||
}
|
|
||||||
|
|
||||||
return serializeArrowTable(data);
|
|
||||||
} else if (payloadType === 'jsontable') {
|
} else if (payloadType === 'jsontable') {
|
||||||
// Serialize array of objects to JSON format
|
// Serialize array of objects to JSON format
|
||||||
if (!Array.isArray(data)) {
|
if (!Array.isArray(data)) {
|
||||||
@@ -154,49 +148,6 @@ async function serializeData(data, payloadType) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper function to properly serialize table data to Arrow IPC
|
|
||||||
* @param {Array<Object>} data - Array of objects representing table rows
|
|
||||||
* @returns {Uint8Array} Arrow IPC formatted buffer
|
|
||||||
*/
|
|
||||||
function serializeArrowTable(data) {
|
|
||||||
if (!Array.isArray(data) || data.length === 0) {
|
|
||||||
throw new Error('Table data must be a non-empty array of objects');
|
|
||||||
}
|
|
||||||
|
|
||||||
logTrace('serializeArrowTable', `Serializing table with ${data.length} rows`);
|
|
||||||
|
|
||||||
// Convert array of objects to a key-value format expected by tableFromArrays
|
|
||||||
const columns = {};
|
|
||||||
const keys = Object.keys(data[0]);
|
|
||||||
for (const key of keys) {
|
|
||||||
columns[key] = data.map(row => row[key]);
|
|
||||||
}
|
|
||||||
|
|
||||||
logTrace('serializeArrowTable', `Columns: ${Object.keys(columns).join(', ')}`);
|
|
||||||
|
|
||||||
const table = tableFromArrays(columns);
|
|
||||||
|
|
||||||
logTrace('serializeArrowTable', `Arrow table created with ${table.numRows} rows, ${table.numCols} cols`);
|
|
||||||
|
|
||||||
// Convert to IPC format
|
|
||||||
const ipcBuffer = tableToIPC(table);
|
|
||||||
|
|
||||||
logTrace('serializeArrowTable', `IPC buffer type: ${typeof ipcBuffer}, byteLength: ${ipcBuffer.byteLength}`);
|
|
||||||
|
|
||||||
const resultBuffer = new Uint8Array(ipcBuffer);
|
|
||||||
logTrace('serializeArrowTable', `Result buffer: ${resultBuffer.length} bytes`);
|
|
||||||
|
|
||||||
// Debug: Show first 20 bytes in hex
|
|
||||||
const hexPreview = [];
|
|
||||||
for (let i = 0; i < Math.min(20, resultBuffer.length); i++) {
|
|
||||||
hexPreview.push(resultBuffer[i].toString(16).padStart(2, '0'));
|
|
||||||
}
|
|
||||||
logTrace('serializeArrowTable', `First 20 bytes (hex): ${hexPreview.join(' ')}`);
|
|
||||||
|
|
||||||
return resultBuffer;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Deserialize bytes to data based on type
|
* Deserialize bytes to data based on type
|
||||||
* @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes
|
* @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes
|
||||||
@@ -210,7 +161,7 @@ async function deserializeData(data, payloadType, correlationId) {
|
|||||||
logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`);
|
logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`);
|
||||||
|
|
||||||
// Debug: Show first 20 bytes in hex for binary data
|
// Debug: Show first 20 bytes in hex for binary data
|
||||||
if (payloadType === 'arrowtable' || payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') {
|
if (payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') {
|
||||||
const hexPreview = [];
|
const hexPreview = [];
|
||||||
for (let i = 0; i < Math.min(20, buffer.length); i++) {
|
for (let i = 0; i < Math.min(20, buffer.length); i++) {
|
||||||
hexPreview.push(buffer[i].toString(16).padStart(2, '0'));
|
hexPreview.push(buffer[i].toString(16).padStart(2, '0'));
|
||||||
@@ -227,18 +178,6 @@ async function deserializeData(data, payloadType, correlationId) {
|
|||||||
const result = JSON.parse(jsonStr);
|
const result = JSON.parse(jsonStr);
|
||||||
logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`);
|
logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`);
|
||||||
return result;
|
return result;
|
||||||
} else if (payloadType === 'arrowtable') {
|
|
||||||
logTrace(correlationId, `deserializeData: Attempting Arrow table deserialization`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Try tableFromIPC (browser API)
|
|
||||||
const table = tableFromIPC(buffer);
|
|
||||||
logTrace(correlationId, `deserializeData: Arrow table from IPC - rows=${table.numRows}, cols=${table.numCols}`);
|
|
||||||
return table;
|
|
||||||
} catch (e) {
|
|
||||||
logTrace(correlationId, `deserializeData: tableFromIPC failed: ${e.message}`);
|
|
||||||
throw new Error(`Unable to deserialize Arrow table: ${e.message}`);
|
|
||||||
}
|
|
||||||
} else if (payloadType === 'jsontable') {
|
} else if (payloadType === 'jsontable') {
|
||||||
const jsonStr = new TextDecoder().decode(buffer);
|
const jsonStr = new TextDecoder().decode(buffer);
|
||||||
const result = JSON.parse(jsonStr);
|
const result = JSON.parse(jsonStr);
|
||||||
@@ -478,8 +417,6 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) {
|
|||||||
let encoding = 'base64';
|
let encoding = 'base64';
|
||||||
if (payloadType === 'jsontable') {
|
if (payloadType === 'jsontable') {
|
||||||
encoding = 'json';
|
encoding = 'json';
|
||||||
} else if (payloadType === 'arrowtable') {
|
|
||||||
encoding = 'arrow-ipc';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
Reference in New Issue
Block a user