architecture.md rev1
This commit is contained in:
@@ -4,6 +4,32 @@
|
||||
|
||||
This document describes the implementation of the high-performance, bi-directional data bridge between Julia and JavaScript services using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
|
||||
|
||||
### Multi-Payload Support
|
||||
|
||||
The implementation uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.**
|
||||
|
||||
**API Standard:**
|
||||
```julia
|
||||
# Input format for smartsend (always a list of tuples)
|
||||
[(dataname1, data1), (dataname2, data2), ...]
|
||||
|
||||
# Output format for smartreceive (always returns a list of tuples)
|
||||
[(dataname1, data1), (dataname2, data2), ...]
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```julia
|
||||
# Single payload - still wrapped in a list
|
||||
smartsend("/test", [(dataname1, data1)], ...)
|
||||
|
||||
# Multiple payloads in one message
|
||||
smartsend("/test", [(dataname1, data1), (dataname2, data2)], ...)
|
||||
|
||||
# Receive always returns a list
|
||||
payloads = smartreceive(msg, ...)
|
||||
# payloads = [(dataname1, data1), (dataname2, data2), ...]
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
The implementation follows the Claim-Check pattern:
|
||||
@@ -107,20 +133,66 @@ node test/scenario3_julia_to_julia.js
|
||||
|
||||
## Usage
|
||||
|
||||
### Scenario 0: Basic Multi-Payload Example
|
||||
|
||||
#### Julia (Sender)
|
||||
```julia
|
||||
using NATSBridge
|
||||
|
||||
# Send multiple payloads in one message
|
||||
smartsend(
|
||||
"/test",
|
||||
[("dataname1", data1), ("dataname2", data2)],
|
||||
nats_url="nats://localhost:4222",
|
||||
fileserver_url="http://localhost:8080/upload",
|
||||
metadata=Dict("custom_key" => "custom_value")
|
||||
)
|
||||
|
||||
# Even single payload must be wrapped in a list
|
||||
smartsend("/test", [("single_data", mydata)])
|
||||
```
|
||||
|
||||
#### Julia (Receiver)
|
||||
```julia
|
||||
using NATSBridge
|
||||
|
||||
# Receive returns a list of payloads
|
||||
payloads = smartreceive(msg, "http://localhost:8080/upload")
|
||||
# payloads = [(dataname1, data1), (dataname2, data2), ...]
|
||||
```
|
||||
|
||||
### Scenario 1: Command & Control (Small JSON)
|
||||
|
||||
#### JavaScript (Sender)
|
||||
```javascript
|
||||
const { SmartSend } = require('./js_bridge');
|
||||
|
||||
const config = {
|
||||
step_size: 0.01,
|
||||
iterations: 1000
|
||||
};
|
||||
// Single payload wrapped in a list
|
||||
const config = [{
|
||||
dataname: "config",
|
||||
data: { step_size: 0.01, iterations: 1000 },
|
||||
type: "json"
|
||||
}];
|
||||
|
||||
await SmartSend("control", config, "json", {
|
||||
correlationId: "unique-id"
|
||||
});
|
||||
|
||||
// Multiple payloads
|
||||
const configs = [
|
||||
{
|
||||
dataname: "config1",
|
||||
data: { step_size: 0.01 },
|
||||
type: "json"
|
||||
},
|
||||
{
|
||||
dataname: "config2",
|
||||
data: { iterations: 1000 },
|
||||
type: "json"
|
||||
}
|
||||
];
|
||||
|
||||
await SmartSend("control", configs, "json");
|
||||
```
|
||||
|
||||
#### Julia (Receiver)
|
||||
@@ -157,8 +229,8 @@ df = DataFrame(
|
||||
category = rand(["A", "B", "C"], 10_000_000)
|
||||
)
|
||||
|
||||
# Send via SmartSend with type="table"
|
||||
await SmartSend("analysis_results", df, "table");
|
||||
# Send via SmartSend - wrapped in a list
|
||||
await SmartSend("analysis_results", [("table_data", df)], "table");
|
||||
```
|
||||
|
||||
#### JavaScript (Receiver)
|
||||
@@ -177,8 +249,12 @@ const table = result.data;
|
||||
```javascript
|
||||
const { SmartSend } = require('./js_bridge');
|
||||
|
||||
// Capture binary chunk
|
||||
const binaryData = await navigator.mediaDevices.getUserMedia({ binary: true });
|
||||
// Binary data wrapped in a list
|
||||
const binaryData = [{
|
||||
dataname: "audio_chunk",
|
||||
data: binaryBuffer,
|
||||
type: "binary"
|
||||
}];
|
||||
|
||||
await SmartSend("binary_input", binaryData, "binary", {
|
||||
metadata: {
|
||||
@@ -208,16 +284,13 @@ end
|
||||
|
||||
#### Julia (Producer)
|
||||
```julia
|
||||
using NATS
|
||||
using NATSBridge
|
||||
|
||||
function publish_health_status(nats)
|
||||
jetstream = JetStream(nats, "health_updates")
|
||||
|
||||
while true
|
||||
status = Dict("cpu" => rand(), "memory" => rand())
|
||||
publish(jetstream, "health", status)
|
||||
sleep(5) # Every 5 seconds
|
||||
end
|
||||
function publish_health_status(nats_url)
|
||||
# Send status wrapped in a list
|
||||
status = Dict("cpu" => rand(), "memory" => rand())
|
||||
smartsend("health", [("status", status)], "json", nats_url=nats_url)
|
||||
sleep(5) # Every 5 seconds
|
||||
end
|
||||
```
|
||||
|
||||
@@ -238,7 +311,8 @@ const consumer = await js.pullSubscribe("health", {
|
||||
// Process historical and real-time messages
|
||||
for await (const msg of consumer) {
|
||||
const result = await SmartReceive(msg);
|
||||
// Process the data
|
||||
// result.data contains the list of payloads
|
||||
// result.envelope contains the message envelope
|
||||
msg.ack();
|
||||
}
|
||||
```
|
||||
@@ -257,16 +331,39 @@ for await (const msg of consumer) {
|
||||
|
||||
```json
|
||||
{
|
||||
"correlation_id": "uuid-v4-string",
|
||||
"type": "json|table|binary",
|
||||
"transport": "direct|link",
|
||||
"payload": "base64-encoded-string", // Only if transport=direct
|
||||
"url": "http://fileserver/path/to/data", // Only if transport=link
|
||||
"correlationId": "uuid-v4-string",
|
||||
"msgId": "uuid-v4-string",
|
||||
"timestamp": "2024-01-15T10:30:00Z",
|
||||
|
||||
"sendTo": "topic/subject",
|
||||
"msgPurpose": "ACK | NACK | updateStatus | shutdown | chat",
|
||||
"senderName": "agent-wine-web-frontend",
|
||||
"senderId": "uuid4",
|
||||
"receiverName": "agent-backend",
|
||||
"receiverId": "uuid4",
|
||||
"replyTo": "topic",
|
||||
"replyToMsgId": "uuid4",
|
||||
"BrokerURL": "nats://localhost:4222",
|
||||
|
||||
"metadata": {
|
||||
"content_type": "application/octet-stream",
|
||||
"content_length": 123456,
|
||||
"format": "arrow_ipc_stream"
|
||||
}
|
||||
"content_length": 123456
|
||||
},
|
||||
|
||||
"payloads": [
|
||||
{
|
||||
"id": "uuid4",
|
||||
"dataname": "login_image",
|
||||
"type": "image",
|
||||
"transport": "direct",
|
||||
"encoding": "base64",
|
||||
"size": 15433,
|
||||
"data": "base64-encoded-string",
|
||||
"metadata": {
|
||||
"checksum": "sha256_hash"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -4,6 +4,60 @@
|
||||
|
||||
This document describes the architecture for a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
|
||||
|
||||
### File Server Handler Architecture
|
||||
|
||||
The system uses **handler functions** to abstract file server operations, allowing support for different file server implementations (e.g., Plik, AWS S3, custom HTTP server).
|
||||
|
||||
**Handler Function Signatures:**
|
||||
|
||||
```julia
|
||||
# Upload handler - uploads data to file server and returns URL
|
||||
fileserverUploadHandler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
|
||||
|
||||
# Download handler - fetches data from file server URL
|
||||
fileserverDownloadHandler(fileserver_url::String, url::String, max_retries::Int, base_delay::Int, max_delay::Int)::Vector{UInt8}
|
||||
```
|
||||
|
||||
This design allows the system to support multiple file server backends without changing the core messaging logic.
|
||||
|
||||
### Multi-Payload Support (Standard API)
|
||||
|
||||
The system uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.**
|
||||
|
||||
**API Standard:**
|
||||
```julia
|
||||
# Input format for smartsend (always a list of tuples)
|
||||
[(dataname1, data1), (dataname2, data2), ...]
|
||||
|
||||
# Output format for smartreceive (always returns a list of tuples)
|
||||
[(dataname1, data1), (dataname2, data2), ...]
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
|
||||
```julia
|
||||
# Single payload - still wrapped in a list
|
||||
smartsend(
|
||||
"/test",
|
||||
[("dataname1", data1)], # List with one tuple
|
||||
nats_url="nats://localhost:4222",
|
||||
fileserverUploadHandler=plik_oneshot_upload,
|
||||
metadata=user_provided_envelope_level_metadata
|
||||
)
|
||||
|
||||
# Multiple payloads in one message
|
||||
smartsend(
|
||||
"/test",
|
||||
[("dataname1", data1), ("dataname2", data2)],
|
||||
nats_url="nats://localhost:4222",
|
||||
fileserverUploadHandler=plik_oneshot_upload
|
||||
)
|
||||
|
||||
# Receive always returns a list
|
||||
payloads = smartreceive(msg, fileserverDownloadHandler, max_retries, base_delay, max_delay)
|
||||
# payloads = [("dataname1", data1), ("dataname2", data2), ...]
|
||||
```
|
||||
|
||||
## Architecture Diagram
|
||||
|
||||
```mermaid
|
||||
@@ -34,30 +88,113 @@ flowchart TD
|
||||
|
||||
## System Components
|
||||
|
||||
### 1. Unified JSON Envelope Schema
|
||||
### 1. msgEnvelope_v1 - Message Envelope
|
||||
|
||||
All messages use a standardized envelope format:
|
||||
The `msgEnvelope_v1` structure provides a comprehensive message format for bidirectional communication between Julia and JavaScript services.
|
||||
|
||||
**Julia Structure:**
|
||||
```julia
|
||||
struct msgEnvelope_v1
|
||||
correlationId::String # Unique identifier to track messages across systems
|
||||
msgId::String # This message id
|
||||
timestamp::String # Message published timestamp
|
||||
|
||||
sendTo::String # Topic/subject the sender sends to
|
||||
msgPurpose::String # Purpose of this message (ACK | NACK | updateStatus | shutdown | ...)
|
||||
senderName::String # Sender name (e.g., "agent-wine-web-frontend")
|
||||
senderId::String # Sender id (uuid4)
|
||||
receiverName::String # Message receiver name (e.g., "agent-backend")
|
||||
receiverId::String # Message receiver id (uuid4 or nothing for broadcast)
|
||||
replyTo::String # Topic to reply to
|
||||
replyToMsgId::String # Message id this message is replying to
|
||||
brokerURL::String # NATS server address
|
||||
|
||||
metadata::Dict{String, Any}
|
||||
payloads::AbstractArray{msgPayload_v1} # Multiple payloads stored here
|
||||
end
|
||||
```
|
||||
|
||||
**JSON Schema:**
|
||||
```json
|
||||
{
|
||||
"correlation_id": "uuid-v4-string",
|
||||
"type": "json|table|binary",
|
||||
"transport": "direct|link",
|
||||
"payload": "base64-encoded-string", // Only if transport=direct
|
||||
"url": "http://fileserver/path/to/data", // Only if transport=link
|
||||
"correlationId": "uuid-v4-string",
|
||||
"msgId": "uuid-v4-string",
|
||||
"timestamp": "2024-01-15T10:30:00Z",
|
||||
|
||||
"sendTo": "topic/subject",
|
||||
"msgPurpose": "ACK | NACK | updateStatus | shutdown | chat",
|
||||
"senderName": "agent-wine-web-frontend",
|
||||
"senderId": "uuid4",
|
||||
"receiverName": "agent-backend",
|
||||
"receiverId": "uuid4",
|
||||
"replyTo": "topic",
|
||||
"replyToMsgId": "uuid4",
|
||||
"brokerURL": "nats://localhost:4222",
|
||||
|
||||
"metadata": {
|
||||
"content_type": "application/octet-stream",
|
||||
"content_length": 123456,
|
||||
"format": "arrow_ipc_stream"
|
||||
}
|
||||
"content_length": 123456
|
||||
},
|
||||
|
||||
"payloads": [
|
||||
{
|
||||
"id": "uuid4",
|
||||
"dataname": "login_image",
|
||||
"type": "image",
|
||||
"transport": "direct",
|
||||
"encoding": "base64",
|
||||
"size": 15433,
|
||||
"data": "base64-encoded-string",
|
||||
"metadata": {
|
||||
"checksum": "sha256_hash"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "uuid4",
|
||||
"dataname": "large_data",
|
||||
"type": "table",
|
||||
"transport": "link",
|
||||
"encoding": "none",
|
||||
"size": 524288,
|
||||
"data": "http://localhost:8080/file/UPLOAD_ID/FILE_ID/data.arrow",
|
||||
"metadata": {
|
||||
"checksum": "sha256_hash"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Transport Strategy Decision Logic
|
||||
### 2. msgPayload_v1 - Payload Structure
|
||||
|
||||
The `msgPayload_v1` structure provides flexible payload handling for various data types.
|
||||
|
||||
**Julia Structure:**
|
||||
```julia
|
||||
struct msgPayload_v1
|
||||
id::String # Id of this payload (e.g., "uuid4")
|
||||
dataname::String # Name of this payload (e.g., "login_image")
|
||||
type::String # "text | json | table | image | audio | video | binary"
|
||||
transport::String # "direct | link"
|
||||
encoding::String # "none | json | base64 | arrow-ipc"
|
||||
size::Integer # Data size in bytes
|
||||
data::Any # Payload data in case of direct transport or a URL in case of link
|
||||
metadata::Dict{String, Any} # Dict("checksum" => "sha256_hash", ...)
|
||||
end
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- Supports multiple data types: text, json, table, image, audio, video, binary
|
||||
- Flexible transport: "direct" (NATS) or "link" (HTTP fileserver)
|
||||
- Multiple payloads per message (essential for chat with mixed content)
|
||||
- Per-payload and per-envelope metadata support
|
||||
|
||||
### 3. Transport Strategy Decision Logic
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ SmartSend Function │
|
||||
│ smartsend Function │
|
||||
│ Accepts: [(dataname1, data1), (dataname2, data2), ...] │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
@@ -65,34 +202,35 @@ All messages use a standardized envelope format:
|
||||
│ Is payload size < 1MB? │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌─────────────────┴─────────────────┐
|
||||
▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐
|
||||
│ Direct Path │ │ Link Path │
|
||||
│ (< 1MB) │ │ (> 1MB) │
|
||||
│ │ │ │
|
||||
│ • Serialize to │ │ • Serialize to │
|
||||
│ IOBuffer │ │ IOBuffer │
|
||||
│ • Base64 encode │ │ • Upload to │
|
||||
│ • Publish to │ │ HTTP Server │
|
||||
│ NATS │ │ • Publish to │
|
||||
│ │ │ NATS with URL │
|
||||
└─────────────────┘ └─────────────────┘
|
||||
┌────────────────┴─-────────────────┐
|
||||
▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐
|
||||
│ Direct Path │ │ Link Path │
|
||||
│ (< 1MB) │ │ (> 1MB) │
|
||||
│ │ │ │
|
||||
│ • Serialize to │ │ • Serialize to │
|
||||
│ IOBuffer │ │ IOBuffer │
|
||||
│ • Base64 encode │ │ • Upload to │
|
||||
│ • Publish to │ │ HTTP Server │
|
||||
│ NATS │ │ • Publish to │
|
||||
│ (with payload │ │ NATS with URL │
|
||||
│ in envelope) │ │ (in envelope) │
|
||||
└─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### 3. Julia Module Architecture
|
||||
### 4. Julia Module Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph JuliaModule
|
||||
SmartSendJulia[SmartSend Julia]
|
||||
smartsendJulia[smartsend Julia]
|
||||
SizeCheck[Size Check]
|
||||
DirectPath[Direct Path]
|
||||
LinkPath[Link Path]
|
||||
HTTPClient[HTTP Client]
|
||||
end
|
||||
|
||||
SmartSendJulia --> SizeCheck
|
||||
smartsendJulia --> SizeCheck
|
||||
SizeCheck -->|< 1MB| DirectPath
|
||||
SizeCheck -->|>= 1MB| LinkPath
|
||||
LinkPath --> HTTPClient
|
||||
@@ -100,19 +238,19 @@ graph TD
|
||||
style JuliaModule fill:#c5e1a5
|
||||
```
|
||||
|
||||
### 4. JavaScript Module Architecture
|
||||
### 5. JavaScript Module Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph JSModule
|
||||
SmartSendJS[SmartSend JS]
|
||||
SmartReceiveJS[SmartReceive JS]
|
||||
smartsendJS[smartsend JS]
|
||||
smartreceiveJS[smartreceive JS]
|
||||
JetStreamConsumer[JetStream Pull Consumer]
|
||||
ApacheArrow[Apache Arrow]
|
||||
end
|
||||
|
||||
SmartSendJS --> NATS
|
||||
SmartReceiveJS --> JetStreamConsumer
|
||||
smartsendJS --> NATS
|
||||
smartreceiveJS --> JetStreamConsumer
|
||||
JetStreamConsumer --> ApacheArrow
|
||||
|
||||
style JSModule fill:#f3e5f5
|
||||
@@ -129,37 +267,64 @@ graph TD
|
||||
- `HTTP.jl` - HTTP client for file server
|
||||
- `Dates.jl` - Timestamps for logging
|
||||
|
||||
#### SmartSend Function
|
||||
#### smartsend Function
|
||||
|
||||
```julia
|
||||
function SmartSend(
|
||||
function smartsend(
|
||||
subject::String,
|
||||
data::Any,
|
||||
data::AbstractArray{Tuple{String, Any}},
|
||||
type::String = "json";
|
||||
nats_url::String = "nats://localhost:4222",
|
||||
fileserver_url::String = "http://localhost:8080/upload",
|
||||
fileserverUploadHandler::Function = plik_oneshot_upload,
|
||||
size_threshold::Int = 1_000_000 # 1MB
|
||||
)
|
||||
```
|
||||
|
||||
**Flow:**
|
||||
1. Serialize data to Arrow IPC stream (if table)
|
||||
2. Check payload size
|
||||
3. If < threshold: publish directly to NATS with Base64-encoded payload
|
||||
4. If >= threshold: upload to HTTP server, publish NATS with URL
|
||||
**Input Format:**
|
||||
- `data::AbstractArray{Tuple{String, Any}}` - **Must be a list of tuples**: `[("dataname1", data1), ("dataname2", data2), ...]`
|
||||
- Even for single payloads: `[(dataname1, data1)]`
|
||||
|
||||
#### SmartReceive Handler
|
||||
**Flow:**
|
||||
1. Iterate through the list of `("dataname", data)` tuples
|
||||
2. For each payload: serialize to Arrow IPC stream (if table) or JSON
|
||||
3. Check payload size
|
||||
4. If < threshold: publish directly to NATS with Base64-encoded payload
|
||||
5. If >= threshold: upload to HTTP server, publish NATS with URL
|
||||
|
||||
#### smartreceive Handler
|
||||
|
||||
```julia
|
||||
function SmartReceive(msg::NATS.Message)
|
||||
function smartreceive(
|
||||
msg::NATS.Message;
|
||||
fileserverDownloadHandler::Function,
|
||||
max_retries::Int = 5,
|
||||
base_delay::Int = 100,
|
||||
max_delay::Int = 5000
|
||||
)
|
||||
# Parse envelope
|
||||
# Check transport type
|
||||
# If direct: decode Base64 payload
|
||||
# If link: fetch from URL with exponential backoff
|
||||
# Deserialize Arrow IPC to DataFrame
|
||||
# Iterate through all payloads
|
||||
# For each payload: check transport type
|
||||
# If direct: decode Base64 payload
|
||||
# If link: fetch from URL with exponential backoff using fileserverDownloadHandler
|
||||
# Deserialize payload based on type
|
||||
# Return list of (dataname, data) tuples
|
||||
end
|
||||
```
|
||||
|
||||
**Output Format:**
|
||||
- Always returns a list of tuples: `[(dataname1, data1), (dataname2, data2), ...]`
|
||||
- Even for single payloads: `[(dataname1, data1)]`
|
||||
|
||||
**Process Flow:**
|
||||
1. Parse the JSON envelope to extract the `payloads` array
|
||||
2. Iterate through each payload in `payloads`
|
||||
3. For each payload:
|
||||
- Determine transport type (`direct` or `link`)
|
||||
- If `direct`: decode Base64 data from the message
|
||||
- If `link`: fetch data from URL using exponential backoff
|
||||
- Deserialize based on payload type (`json`, `table`, `binary`, etc.)
|
||||
4. Return list of `(dataname, data)` tuples
|
||||
|
||||
### JavaScript Implementation
|
||||
|
||||
#### Dependencies
|
||||
@@ -167,10 +332,13 @@ end
|
||||
- `apache-arrow` - Arrow IPC serialization
|
||||
- `uuid` - Correlation ID generation
|
||||
|
||||
#### SmartSend Function
|
||||
#### smartsend Function
|
||||
|
||||
```javascript
|
||||
async function SmartSend(subject, data, type = 'json', options = {})
|
||||
async function smartsend(subject, data, type = 'json', options = {})
|
||||
// options object should include:
|
||||
// - fileserverUploadHandler: function to upload data to file server
|
||||
// - fileserver_url: base URL of the file server
|
||||
```
|
||||
|
||||
**Flow:**
|
||||
@@ -179,18 +347,27 @@ async function SmartSend(subject, data, type = 'json', options = {})
|
||||
3. If < threshold: publish directly to NATS
|
||||
4. If >= threshold: upload to HTTP server, publish NATS with URL
|
||||
|
||||
#### SmartReceive Handler
|
||||
#### smartreceive Handler
|
||||
|
||||
```javascript
|
||||
async function SmartReceive(msg, options = {})
|
||||
async function smartreceive(msg, options = {})
|
||||
// options object should include:
|
||||
// - fileserverDownloadHandler: function to fetch data from file server URL
|
||||
// - fileserver_url: base URL of the file server
|
||||
// - max_retries: maximum retry attempts for fetching URL
|
||||
// - base_delay: initial delay for exponential backoff in ms
|
||||
// - max_delay: maximum delay for exponential backoff in ms
|
||||
```
|
||||
|
||||
**Flow:**
|
||||
1. Parse envelope
|
||||
2. Check transport type
|
||||
3. If direct: decode Base64 payload
|
||||
4. If link: fetch with exponential backoff
|
||||
5. Deserialize Arrow IPC with zero-copy
|
||||
**Process Flow:**
|
||||
1. Parse the JSON envelope to extract the `payloads` array
|
||||
2. Iterate through each payload in `payloads`
|
||||
3. For each payload:
|
||||
- Determine transport type (`direct` or `link`)
|
||||
- If `direct`: decode Base64 data from the message
|
||||
- If `link`: fetch data from URL using exponential backoff
|
||||
- Deserialize based on payload type (`json`, `table`, `binary`, etc.)
|
||||
4. Return list of `(dataname, data)` tuples
|
||||
|
||||
## Scenario Implementations
|
||||
|
||||
@@ -207,7 +384,7 @@ async function SmartReceive(msg, options = {})
|
||||
**JavaScript (Sender):**
|
||||
```javascript
|
||||
// Create small JSON config
|
||||
// Send via SmartSend with type="json"
|
||||
// Send via smartsend with type="json"
|
||||
```
|
||||
|
||||
### Scenario 2: Deep Dive Analysis (Large Arrow Table)
|
||||
@@ -235,7 +412,7 @@ async function SmartReceive(msg, options = {})
|
||||
```javascript
|
||||
// Capture audio chunk
|
||||
// Send as binary with metadata headers
|
||||
// Use SmartSend with type="audio"
|
||||
// Use smartsend with type="audio"
|
||||
```
|
||||
|
||||
**Julia (Receiver):**
|
||||
@@ -260,6 +437,76 @@ async function SmartReceive(msg, options = {})
|
||||
// Process historical and real-time messages
|
||||
```
|
||||
|
||||
### Scenario 5: Selection (Low Bandwidth)
|
||||
|
||||
**Focus:** Small Arrow tables, Julia to JavaScript. The Action: Julia wants to send a small DataFrame to show on a JavaScript dashboard for the user to choose.
|
||||
|
||||
**Julia (Sender):**
|
||||
```julia
|
||||
# Create small DataFrame (e.g., 50KB - 500KB)
|
||||
# Convert to Arrow IPC stream
|
||||
# Check payload size (< 1MB threshold)
|
||||
# Publish directly to NATS with Base64-encoded payload
|
||||
# Include metadata for dashboard selection context
|
||||
```
|
||||
|
||||
**JavaScript (Receiver):**
|
||||
```javascript
|
||||
// Receive NATS message with direct transport
|
||||
// Decode Base64 payload
|
||||
// Parse Arrow IPC with zero-copy
|
||||
// Load into selection UI component (e.g., dropdown, table)
|
||||
// User makes selection
|
||||
// Send selection back to Julia
|
||||
```
|
||||
|
||||
**Use Case:** Julia server generates a list of available options (e.g., file selections, configuration presets) as a small DataFrame and sends to JavaScript dashboard for user selection. The selection is then sent back to Julia for processing.
|
||||
|
||||
### Scenario 6: Chat System
|
||||
|
||||
**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging.
|
||||
|
||||
**Multi-Payload Support:** The system supports mixed-payload messages where a single message can contain multiple payloads with different transport strategies. The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type.
|
||||
|
||||
**Julia (Sender/Receiver):**
|
||||
```julia
|
||||
# Build chat message with mixed payloads:
|
||||
# - Text: direct transport (Base64)
|
||||
# - Small images: direct transport (Base64)
|
||||
# - Large images: link transport (HTTP URL)
|
||||
# - Audio/video: link transport (HTTP URL)
|
||||
# - Tables: direct or link depending on size
|
||||
# - Files: link transport (HTTP URL)
|
||||
#
|
||||
# Each payload uses appropriate transport strategy:
|
||||
# - Size < 1MB → direct (NATS + Base64)
|
||||
# - Size >= 1MB → link (HTTP upload + NATS URL)
|
||||
#
|
||||
# Include claim-check metadata for delivery tracking
|
||||
# Support bidirectional messaging with replyTo fields
|
||||
```
|
||||
|
||||
**JavaScript (Sender/Receiver):**
|
||||
```javascript
|
||||
// Build chat message with mixed content:
|
||||
// - User input text: direct transport
|
||||
// - Selected image: check size, use appropriate transport
|
||||
// - Audio recording: link transport for large files
|
||||
// - File attachment: link transport
|
||||
//
|
||||
// Parse received message:
|
||||
// - Direct payloads: decode Base64
|
||||
// - Link payloads: fetch from HTTP with exponential backoff
|
||||
// - Deserialize all payloads appropriately
|
||||
//
|
||||
// Render mixed content in chat interface
|
||||
// Support bidirectional reply with claim-check delivery confirmation
|
||||
```
|
||||
|
||||
**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components.
|
||||
|
||||
**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msgEnvelope_v1` supports `AbstractArray{msgPayload_v1}` for multiple payloads.
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### Zero-Copy Reading
|
||||
@@ -280,8 +527,8 @@ async function SmartReceive(msg, options = {})
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
- Test SmartSend with various payload sizes
|
||||
- Test SmartReceive with direct and link transport
|
||||
- Test smartsend with various payload sizes
|
||||
- Test smartreceive with direct and link transport
|
||||
- Test Arrow IPC serialization/deserialization
|
||||
|
||||
### Integration Tests
|
||||
|
||||
@@ -18,12 +18,12 @@ const DEFAULT_FILESERVER_URL = "http://localhost:8080/upload" # Default HTTP fi
|
||||
struct msgPayload_v1
|
||||
id::String # id of this payload e.g. "uuid4"
|
||||
dataname::String # name of this payload e.g. "login_image"
|
||||
type::String # "text | json | table | image | audio | video | binary"
|
||||
type::String # this payload type. Can be "text | json | table | image | audio | video | binary"
|
||||
transport::String # "direct | link"
|
||||
encoding::String # "none | json | base64 | arrow-ipc"
|
||||
size::Integer # data size in bytes e.g. 15433
|
||||
data::Any # payload data in case of direct transport or a URL in case of link
|
||||
metadata::Dict{String, Any} # Dict("checksum=> "sha256_hash", ...)
|
||||
metadata::Dict{String, Any} # Dict("checksum=> "sha256_hash", ...) This metadata is for this payload
|
||||
end
|
||||
|
||||
# constructor
|
||||
@@ -56,15 +56,15 @@ struct msgEnvelope_v1
|
||||
timestamp::String # message published timestamp. string(Dates.now())
|
||||
|
||||
sendTo::String # topic/subject the sender sends to e.g. "/agent/wine/api/v1/prompt"
|
||||
msgPurpose::String # purpose of this message e.g. "ACK | NACK | updateStatus | shutdown | ..."
|
||||
senderName::String # sender name (String) e.g. "agent-wine-web-frontend"
|
||||
msgPurpose::String # purpose of this message e.g. "ACK | NACK | updateStatus | shutdown | ..."
|
||||
senderName::String # sender name (String) e.g. "agent-wine-web-frontend"
|
||||
senderId::String # sender id e.g. uuid4snakecase()
|
||||
receiverName::String # msg receiver name (String) e.g. "agent-backend"
|
||||
receiverId::String # msg receiver id, nothing means everyone in the topic e.g. uuid4snakecase()
|
||||
|
||||
replyTo::String # sender ask receiver to reply to this topic
|
||||
replyToMsgId::String # the message id this message is replying to
|
||||
BrokerURL::String # mqtt/NATS server address
|
||||
brokerURL::String # mqtt/NATS server address
|
||||
|
||||
metadata::Dict{String, Any}
|
||||
payloads::AbstractArray{msgPayload_v1} # multiple payload store here
|
||||
@@ -83,7 +83,7 @@ function msgEnvelope_v1(
|
||||
receiverId::String = "",
|
||||
replyTo::String = "",
|
||||
replyToMsgId::String = "",
|
||||
BrokerURL::String = DEFAULT_NATS_URL,
|
||||
brokerURL::String = DEFAULT_NATS_URL,
|
||||
metadata::Dict{String, Any} = Dict{String, Any}(),
|
||||
payloads::AbstractArray{msgPayload_v1} = msgPayload_v1[]
|
||||
)
|
||||
@@ -99,7 +99,7 @@ function msgEnvelope_v1(
|
||||
receiverId,
|
||||
replyTo,
|
||||
replyToMsgId,
|
||||
BrokerURL,
|
||||
brokerURL,
|
||||
metadata,
|
||||
payloads
|
||||
)
|
||||
@@ -107,94 +107,74 @@ end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
""" Struct for the unified JSON envelope
|
||||
This struct represents a standardized message format that can carry either
|
||||
direct payload data or a URL reference, allowing flexible transport strategies
|
||||
based on payload size and requirements.
|
||||
""" Convert msgEnvelope_v1 to JSON string
|
||||
This function converts the msgEnvelope_v1 struct to a JSON string representation.
|
||||
"""
|
||||
struct MessageEnvelope
|
||||
correlation_id::String # Unique identifier to track messages across systems
|
||||
type::String # Data type indicator (e.g., "json", "table", "binary")
|
||||
transport::String # Transport strategy: "direct" (base64 encoded bytes) or "link" (URL reference)
|
||||
payload::Union{String, Nothing} # Base64-encoded payload for direct transport
|
||||
url::Union{String, Nothing} # URL reference for link transport
|
||||
metadata::Dict{String, Any} # Additional metadata about the payload
|
||||
end
|
||||
|
||||
""" Constructor for MessageEnvelope with keyword arguments and defaults
|
||||
This constructor provides a convenient way to create an envelope using keyword arguments,
|
||||
automatically generating a correlation ID if not provided, and defaulting to "json" type
|
||||
and "direct" transport.
|
||||
"""
|
||||
function MessageEnvelope(
|
||||
; correlation_id::String = string(uuid4()), # Generate unique ID if not provided
|
||||
type::String = "json", # Default data type
|
||||
transport::String = "direct", # Default transport method
|
||||
payload::Union{String, Nothing} = nothing, # No payload by default
|
||||
url::Union{String, Nothing} = nothing, # No URL by default
|
||||
metadata::Dict{String, Any} = Dict{String, Any}() # Empty metadata by default
|
||||
)
|
||||
MessageEnvelope(correlation_id, type, transport, payload, url, metadata)
|
||||
end
|
||||
|
||||
""" Constructor for MessageEnvelope from JSON string
|
||||
This constructor parses a JSON string and reconstructs a MessageEnvelope struct.
|
||||
It handles the metadata field specially by converting the JSON object to a Julia Dict,
|
||||
extracting values from the JSON structure for all other fields.
|
||||
"""
|
||||
function MessageEnvelope(json_str::String)
|
||||
data = JSON.parse(json_str) # Parse JSON string into Julia data structure
|
||||
metadata = Dict{String, Any}()
|
||||
if haskey(data, :metadata) # Check if metadata exists in JSON
|
||||
metadata = Dict(String(k) => v for (k, v) in data.metadata) # Convert JSON keys to strings and store in Dict
|
||||
end
|
||||
|
||||
MessageEnvelope(
|
||||
correlation_id = String(data.correlation_id), # Extract correlation_id from JSON data
|
||||
type = String(data.type), # Extract type from JSON data
|
||||
transport = String(data.transport), # Extract transport from JSON data
|
||||
payload = haskey(data, :payload) ? String(data.payload) : nothing, # Extract payload if present
|
||||
url = haskey(data, :url) ? String(data.url) : nothing, # Extract URL if present
|
||||
metadata = metadata # Use the parsed metadata
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
""" Convert MessageEnvelope to JSON string
|
||||
This function converts the MessageEnvelope struct to a JSON string representation.
|
||||
It only includes fields in the JSON output if they have non-nothing values,
|
||||
making the JSON output cleaner and more efficient.
|
||||
"""
|
||||
function envelope_to_json(env::MessageEnvelope)
|
||||
function envelope_to_json(env::msgEnvelope_v1)
|
||||
obj = Dict{String, Any}(
|
||||
"correlation_id" => env.correlation_id, # Always include correlation_id
|
||||
"type" => env.type, # Always include type
|
||||
"transport" => env.transport # Always include transport
|
||||
"correlationId" => env.correlationId,
|
||||
"msgId" => env.msgId,
|
||||
"timestamp" => env.timestamp,
|
||||
"sendTo" => env.sendTo,
|
||||
"msgPurpose" => env.msgPurpose,
|
||||
"senderName" => env.senderName,
|
||||
"senderId" => env.senderId,
|
||||
"receiverName" => env.receiverName,
|
||||
"receiverId" => env.receiverId,
|
||||
"replyTo" => env.replyTo,
|
||||
"replyToMsgId" => env.replyToMsgId,
|
||||
"brokerURL" => env.brokerURL
|
||||
)
|
||||
|
||||
if env.payload !== nothing # Only include payload if it exists
|
||||
obj["payload"] = env.payload
|
||||
end
|
||||
|
||||
if env.url !== nothing # Only include URL if it exists
|
||||
obj["url"] = env.url
|
||||
end
|
||||
|
||||
if !isempty(env.metadata) # Only include metadata if it exists and is not empty
|
||||
obj["metadata"] = env.metadata
|
||||
end
|
||||
|
||||
JSON.json(obj) # Convert Dict to JSON string
|
||||
|
||||
if !isempty(env.metadata) # Only include metadata if it exists and is not empty
|
||||
obj["metadata"] = Dict(String(k) => v for (k, v) in env.metadata)
|
||||
end
|
||||
|
||||
# Convert payloads to JSON array
|
||||
if !isempty(env.payloads)
|
||||
payloads_json = []
|
||||
for payload in env.payloads
|
||||
payload_obj = Dict{String, Any}(
|
||||
"id" => payload.id,
|
||||
"dataname" => payload.dataname,
|
||||
"type" => payload.type,
|
||||
"transport" => payload.transport,
|
||||
"encoding" => payload.encoding,
|
||||
"size" => payload.size
|
||||
)
|
||||
# Include data based on transport type
|
||||
if payload.transport == "direct" && payload.data !== nothing
|
||||
if payload.encoding == "base64" || payload.encoding == "json"
|
||||
payload_obj["data"] = payload.data
|
||||
else
|
||||
# For other encodings, use base64
|
||||
payload_bytes = _get_payload_bytes(payload.data)
|
||||
payload_obj["data"] = Base64.base64encode(payload_bytes)
|
||||
end
|
||||
end
|
||||
if !isempty(payload.metadata)
|
||||
payload_obj["metadata"] = Dict(String(k) => v for (k, v) in payload.metadata)
|
||||
end
|
||||
push!(payloads_json, payload_obj)
|
||||
end
|
||||
obj["payloads"] = payloads_json
|
||||
end
|
||||
|
||||
JSON.json(obj)
|
||||
end
|
||||
|
||||
""" Helper function to get payload bytes from data
|
||||
"""
|
||||
function _get_payload_bytes(data::Any)
|
||||
# This is a placeholder - actual implementation depends on data type
|
||||
if isa(data, Vector{UInt8})
|
||||
return data
|
||||
elseif isa(data, String)
|
||||
return bytes(data)
|
||||
else
|
||||
return String(data)
|
||||
end
|
||||
end
|
||||
|
||||
""" Log a trace message with correlation ID and timestamp
|
||||
This function logs information messages with a correlation ID for tracing purposes,
|
||||
@@ -215,8 +195,8 @@ Otherwise, it uploads the data to a fileserver (by default using `plik_oneshot_u
|
||||
The function workflow:
|
||||
1. Serializes the provided data according to the specified format (`type`)
|
||||
2. Compares the serialized size against `size_threshold`
|
||||
3. For small payloads: encodes as Base64, constructs a "direct" MessageEnvelope, and publishes to NATS
|
||||
4. For large payloads: uploads to the fileserver, constructs a "link" MessageEnvelope with the URL, and publishes to NATS
|
||||
3. For small payloads: encodes as Base64, constructs a "direct" msgEnvelope_v1, and publishes to NATS
|
||||
4. For large payloads: uploads to the fileserver, constructs a "link" msgEnvelope_v1 with the URL, and publishes to NATS
|
||||
|
||||
# Arguments:
|
||||
- `subject::String` - NATS subject to publish the message to
|
||||
@@ -230,15 +210,15 @@ The function workflow:
|
||||
- `fileServerUploadHandler::Function = plik_oneshot_upload` - Function to handle fileserver uploads (must match signature of `plik_oneshot_upload`)
|
||||
- `size_threshold::Int = DEFAULT_SIZE_THRESHOLD` - Threshold in bytes separating direct vs link transport
|
||||
- `correlation_id::Union{String, Nothing} = nothing` - Optional correlation ID for tracing; if `nothing`, a UUID is generated
|
||||
- `msg_purpose::String = "chat"` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc.
|
||||
- `sender_name::String = "NATSBridge"` - Name of the sender
|
||||
- `receiver_name::String = ""` - Name of the receiver (empty string means broadcast)
|
||||
- `receiver_id::String = ""` - UUID of the receiver (empty string means broadcast)
|
||||
- `reply_to::String = ""` - Topic to reply to (empty string if no reply expected)
|
||||
- `reply_to_msg_id::String = ""` - Message ID this message is replying to
|
||||
|
||||
# Return:
|
||||
- A `MessageEnvelope` object containing metadata and transport information:
|
||||
- `correlation_id::String` - Unique identifier for this message exchange
|
||||
- `type::String` - Serialization type used (`"json"` or `"arrow"`)
|
||||
- `transport::String` - Either `"direct"` or `"link"`
|
||||
- `payload::Union{String, Nothing}` - Base64-encoded data for direct transport, `nothing` for link transport
|
||||
- `url::Union{String, Nothing}` - Download URL for link transport, `nothing` for direct transport
|
||||
- `metadata::Dict` - Additional metadata (e.g., `"content_length"`, `"format"`)
|
||||
- A `msgEnvelope_v1` object containing metadata and transport information
|
||||
|
||||
# Example
|
||||
```julia
|
||||
@@ -251,11 +231,6 @@ env = smartsend("my.subject", data, "json")
|
||||
# Send a large array using fileserver upload
|
||||
data = rand(10_000_000) # ~80 MB
|
||||
env = smartsend("large.data", data, "arrow")
|
||||
|
||||
# In another process, retrieve and deserialize:
|
||||
# msg = subscribe(nats_url, "my.subject")
|
||||
# env = json_to_envelope(msg.data)
|
||||
# data = _deserialize_data(Base64.decode(env.payload), env.type)
|
||||
```
|
||||
"""
|
||||
function smartsend(
|
||||
@@ -267,7 +242,13 @@ function smartsend(
|
||||
fileserver_url::String = DEFAULT_FILESERVER_URL,
|
||||
fileServerUploadHandler::Function=plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver
|
||||
size_threshold::Int = DEFAULT_SIZE_THRESHOLD,
|
||||
correlation_id::Union{String, Nothing} = nothing
|
||||
correlation_id::Union{String, Nothing} = nothing,
|
||||
msg_purpose::String = "chat",
|
||||
sender_name::String = "NATSBridge",
|
||||
receiver_name::String = "",
|
||||
receiver_id::String = "",
|
||||
reply_to::String = "",
|
||||
reply_to_msg_id::String = ""
|
||||
)
|
||||
# Generate correlation ID if not provided
|
||||
cid = correlation_id !== nothing ? correlation_id : string(uuid4()) # Create or use provided correlation ID
|
||||
@@ -280,51 +261,99 @@ function smartsend(
|
||||
payload_size = length(payload_bytes) # Calculate payload size in bytes
|
||||
log_trace(cid, "Serialized payload size: $payload_size bytes") # Log payload size
|
||||
|
||||
# Decision: Direct vs Link
|
||||
if payload_size < size_threshold # Check if payload is small enough for direct transport
|
||||
# Direct path - Base64 encode and send via NATS
|
||||
payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string
|
||||
log_trace(cid, "Using direct transport for $payload_size bytes") # Log transport choice
|
||||
|
||||
env = MessageEnvelope( # Create envelope for direct transport
|
||||
correlation_id = cid,
|
||||
type = type,
|
||||
transport = "direct",
|
||||
payload = payload_b64,
|
||||
metadata = Dict("dataname" => dataname, "content_length" => payload_size, "format" => "arrow_ipc_stream")
|
||||
)
|
||||
|
||||
msg_json = envelope_to_json(env) # Convert envelope to JSON
|
||||
publish_message(nats_url, subject, msg_json, cid) # Publish message to NATS
|
||||
|
||||
return env # Return the envelope for tracking
|
||||
else
|
||||
# Link path - Upload to HTTP server, send URL via NATS
|
||||
log_trace(cid, "Using link transport, uploading to fileserver") # Log link transport choice
|
||||
|
||||
# Upload to HTTP server
|
||||
response = fileServerUploadHandler(fileserver_url, dataname, payload_bytes)
|
||||
|
||||
if response[:status] != 200 # Check if upload was successful
|
||||
error("Failed to upload data to fileserver: $(response[:status])") # Throw error if upload failed
|
||||
end
|
||||
|
||||
url = response[:url] # URL for the uploaded data
|
||||
log_trace(cid, "Uploaded to URL: $url") # Log successful upload
|
||||
# Generate unique IDs
|
||||
msg_id = string(uuid4())
|
||||
timestamp = string(Dates.now())
|
||||
|
||||
# Decision: Direct vs Link
|
||||
if payload_size < size_threshold # Check if payload is small enough for direct transport
|
||||
# Direct path - Base64 encode and send via NATS
|
||||
payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string
|
||||
log_trace(cid, "Using direct transport for $payload_size bytes") # Log transport choice
|
||||
|
||||
# Create msgPayload_v1 for direct transport
|
||||
payload = msgPayload_v1(
|
||||
id = string(uuid4()),
|
||||
dataname = dataname,
|
||||
type = type,
|
||||
transport = "direct",
|
||||
encoding = "base64",
|
||||
size = payload_size,
|
||||
data = payload_b64,
|
||||
metadata = Dict("dataname" => dataname, "content_length" => payload_size, "format" => "arrow_ipc_stream")
|
||||
)
|
||||
|
||||
# Create msgEnvelope_v1 with all fields populated
|
||||
env = msgEnvelope_v1(
|
||||
correlationId = cid,
|
||||
msgId = msg_id,
|
||||
timestamp = timestamp,
|
||||
sendTo = subject,
|
||||
msgPurpose = msg_purpose,
|
||||
senderName = sender_name,
|
||||
senderId = string(uuid4()),
|
||||
receiverName = receiver_name,
|
||||
receiverId = receiver_id,
|
||||
replyTo = reply_to,
|
||||
replyToMsgId = reply_to_msg_id,
|
||||
brokerURL = nats_url,
|
||||
metadata = Dict(),
|
||||
payloads = [payload]
|
||||
)
|
||||
|
||||
msg_json = envelope_to_json(env) # Convert envelope to JSON
|
||||
publish_message(nats_url, subject, msg_json, cid) # Publish message to NATS
|
||||
|
||||
return env # Return the envelope for tracking
|
||||
else
|
||||
# Link path - Upload to HTTP server, send URL via NATS
|
||||
log_trace(cid, "Using link transport, uploading to fileserver") # Log link transport choice
|
||||
|
||||
# Upload to HTTP server
|
||||
response = fileServerUploadHandler(fileserver_url, dataname, payload_bytes)
|
||||
|
||||
if response[:status] != 200 # Check if upload was successful
|
||||
error("Failed to upload data to fileserver: $(response[:status])") # Throw error if upload failed
|
||||
end
|
||||
|
||||
url = response[:url] # URL for the uploaded data
|
||||
log_trace(cid, "Uploaded to URL: $url") # Log successful upload
|
||||
|
||||
env = MessageEnvelope( # Create envelope for link transport
|
||||
correlation_id = cid,
|
||||
type = type,
|
||||
transport = "link",
|
||||
url = url,
|
||||
metadata = Dict("dataname" => dataname, "content_length" => payload_size, "format" => "arrow_ipc_stream")
|
||||
)
|
||||
|
||||
msg_json = envelope_to_json(env) # Convert envelope to JSON
|
||||
publish_message(nats_url, subject, msg_json, cid) # Publish message to NATS
|
||||
|
||||
return env # Return the envelope for tracking
|
||||
end
|
||||
# Create msgPayload_v1 for link transport
|
||||
payload = msgPayload_v1(
|
||||
id = string(uuid4()),
|
||||
dataname = dataname,
|
||||
type = type,
|
||||
transport = "link",
|
||||
encoding = "none",
|
||||
size = payload_size,
|
||||
data = url,
|
||||
metadata = Dict("dataname" => dataname, "content_length" => payload_size, "format" => "arrow_ipc_stream")
|
||||
)
|
||||
|
||||
# Create msgEnvelope_v1 with all fields populated
|
||||
env = msgEnvelope_v1(
|
||||
correlationId = cid,
|
||||
msgId = msg_id,
|
||||
timestamp = timestamp,
|
||||
sendTo = subject,
|
||||
msgPurpose = msg_purpose,
|
||||
senderName = sender_name,
|
||||
senderId = string(uuid4()),
|
||||
receiverName = receiver_name,
|
||||
receiverId = receiver_id,
|
||||
replyTo = reply_to,
|
||||
replyToMsgId = reply_to_msg_id,
|
||||
brokerURL = nats_url,
|
||||
metadata = Dict(),
|
||||
payloads = [payload]
|
||||
)
|
||||
|
||||
msg_json = envelope_to_json(env) # Convert envelope to JSON
|
||||
publish_message(nats_url, subject, msg_json, cid) # Publish message to NATS
|
||||
|
||||
return env # Return the envelope for tracking
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -436,7 +465,7 @@ Keyword Arguments:
|
||||
- `max_delay::Int` - Maximum delay for exponential backoff in ms (default: 5000)
|
||||
|
||||
Return:
|
||||
- Tuple `(data = deserialized_data, envelope = MessageEnvelope)` - Data and envelope
|
||||
- Tuple `(data = deserialized_data, envelope = msgEnvelope_v1)` - Data and envelope
|
||||
"""
|
||||
function smartreceive(
|
||||
msg::NATS.Msg;
|
||||
@@ -445,33 +474,79 @@ function smartreceive(
|
||||
base_delay::Int = 100,
|
||||
max_delay::Int = 5000
|
||||
)
|
||||
# Parse the envelope
|
||||
env = MessageEnvelope(String(msg.payload)) # Parse NATS message data as JSON envelope
|
||||
log_trace(env.correlation_id, "Processing received message") # Log message processing start
|
||||
# Parse the JSON envelope
|
||||
json_data = JSON.parse(String(msg.payload))
|
||||
|
||||
# Check transport type
|
||||
if env.transport == "direct" # Direct transport - payload is in the message
|
||||
log_trace(env.correlation_id, "Direct transport - decoding payload") # Log direct transport handling
|
||||
# Get transport from the first payload
|
||||
transport = String(json_data["payloads"][1]["transport"])
|
||||
log_trace(json_data["correlationId"], "Processing received message") # Log message processing start
|
||||
|
||||
if transport == "direct" # Direct transport - payload is in the message
|
||||
log_trace(json_data["correlationId"], "Direct transport - decoding payload") # Log direct transport handling
|
||||
|
||||
# Extract base64 payload from the first payload
|
||||
payload_b64 = String(json_data["payloads"][1]["data"])
|
||||
|
||||
# Decode Base64 payload
|
||||
payload_bytes = Base64.base64decode(env.payload) # Decode base64 payload to bytes
|
||||
payload_bytes = Base64.base64decode(payload_b64) # Decode base64 payload to bytes
|
||||
|
||||
# Deserialize based on type
|
||||
data = _deserialize_data(payload_bytes, env.type, env.correlation_id, env.metadata) # Convert bytes to Julia data
|
||||
|
||||
data_type = String(json_data["payloads"][1]["type"])
|
||||
data = _deserialize_data(payload_bytes, data_type, json_data["correlationId"], Dict{String, Any}())
|
||||
|
||||
# Create msgEnvelope_v1 from parsed data
|
||||
env = msgEnvelope_v1(
|
||||
correlationId = json_data["correlationId"],
|
||||
msgId = haskey(json_data, "msgId") ? String(json_data["msgId"]) : "",
|
||||
timestamp = haskey(json_data, "timestamp") ? String(json_data["timestamp"]) : "",
|
||||
sendTo = json_data["sendTo"],
|
||||
msgPurpose = haskey(json_data, "msgPurpose") ? String(json_data["msgPurpose"]) : "",
|
||||
senderName = haskey(json_data, "senderName") ? String(json_data["senderName"]) : "",
|
||||
senderId = haskey(json_data, "senderId") ? String(json_data["senderId"]) : "",
|
||||
receiverName = haskey(json_data, "receiverName") ? String(json_data["receiverName"]) : "",
|
||||
receiverId = haskey(json_data, "receiverId") ? String(json_data["receiverId"]) : "",
|
||||
replyTo = haskey(json_data, "replyTo") ? String(json_data["replyTo"]) : "",
|
||||
replyToMsgId = haskey(json_data, "replyToMsgId") ? String(json_data["replyToMsgId"]) : "",
|
||||
brokerURL = haskey(json_data, "brokerURL") ? String(json_data["brokerURL"]) : DEFAULT_NATS_URL,
|
||||
metadata = Dict{String, Any}(),
|
||||
payloads = msgPayload_v1[]
|
||||
)
|
||||
|
||||
return (data = data, envelope = env) # Return data and envelope as tuple
|
||||
elseif env.transport == "link" # Link transport - payload is at URL
|
||||
log_trace(env.correlation_id, "Link transport - fetching from URL") # Log link transport handling
|
||||
elseif transport == "link" # Link transport - payload is at URL
|
||||
log_trace(json_data["correlationId"], "Link transport - fetching from URL") # Log link transport handling
|
||||
|
||||
# Extract URL from the first payload
|
||||
url = String(json_data["payloads"][1]["data"])
|
||||
|
||||
# Fetch with exponential backoff
|
||||
downloaded_data = _fetch_with_backoff(env.url, max_retries, base_delay, max_delay, env.correlation_id) # Fetch data from URL
|
||||
downloaded_data = _fetch_with_backoff(url, max_retries, base_delay, max_delay, json_data["correlationId"]) # Fetch data from URL
|
||||
|
||||
# Deserialize based on type
|
||||
data = _deserialize_data(downloaded_data, env.type, env.correlation_id, env.metadata) # Convert bytes to Julia data
|
||||
data_type = String(json_data["payloads"][1]["type"])
|
||||
data = _deserialize_data(downloaded_data, data_type, json_data["correlationId"], Dict{String, Any}())
|
||||
|
||||
# Create msgEnvelope_v1 from parsed data
|
||||
env = msgEnvelope_v1(
|
||||
correlationId = json_data["correlationId"],
|
||||
msgId = haskey(json_data, "msgId") ? String(json_data["msgId"]) : "",
|
||||
timestamp = haskey(json_data, "timestamp") ? String(json_data["timestamp"]) : "",
|
||||
sendTo = json_data["sendTo"],
|
||||
msgPurpose = haskey(json_data, "msgPurpose") ? String(json_data["msgPurpose"]) : "",
|
||||
senderName = haskey(json_data, "senderName") ? String(json_data["senderName"]) : "",
|
||||
senderId = haskey(json_data, "senderId") ? String(json_data["senderId"]) : "",
|
||||
receiverName = haskey(json_data, "receiverName") ? String(json_data["receiverName"]) : "",
|
||||
receiverId = haskey(json_data, "receiverId") ? String(json_data["receiverId"]) : "",
|
||||
replyTo = haskey(json_data, "replyTo") ? String(json_data["replyTo"]) : "",
|
||||
replyToMsgId = haskey(json_data, "replyToMsgId") ? String(json_data["replyToMsgId"]) : "",
|
||||
brokerURL = haskey(json_data, "brokerURL") ? String(json_data["brokerURL"]) : DEFAULT_NATS_URL,
|
||||
metadata = Dict{String, Any}(),
|
||||
payloads = msgPayload_v1[]
|
||||
)
|
||||
|
||||
return (data = data, envelope = env) # Return data and envelope as tuple
|
||||
else # Unknown transport type
|
||||
error("Unknown transport type: $(env.transport)") # Throw error for unknown transport
|
||||
error("Unknown transport type: $(transport)") # Throw error for unknown transport
|
||||
end
|
||||
end
|
||||
|
||||
@@ -556,21 +631,6 @@ function _deserialize_data(
|
||||
end
|
||||
|
||||
|
||||
# """ Decode base64 string to bytes
|
||||
# This internal function decodes a base64-encoded string back to binary data.
|
||||
# It's a wrapper around Base64.decode for consistency in the module.
|
||||
|
||||
# Arguments:
|
||||
# - `str::String` - Base64-encoded string to decode
|
||||
|
||||
# Return:
|
||||
# - Vector{UInt8} - Decoded binary data
|
||||
# """
|
||||
# function base64decode(str::String)
|
||||
# return Base64.decode(str) # Decode base64 string to bytes using Julia's Base64 module
|
||||
# end
|
||||
|
||||
|
||||
""" plik_oneshot_upload - Upload a single file to a plik server using one-shot mode
|
||||
|
||||
This function uploads a raw byte array to a plik server in one-shot mode (no upload session).
|
||||
@@ -609,7 +669,7 @@ status, uploadid, fileid, url = plik_oneshot_upload(fileServerURL, filename, fil
|
||||
# to download an uploaded file
|
||||
curl -L -O "url"
|
||||
```
|
||||
""" #[x]
|
||||
"""
|
||||
function plik_oneshot_upload(fileServerURL::String, filename::String, data::Vector{UInt8})
|
||||
|
||||
# ----------------------------------------- get upload id ---------------------------------------- #
|
||||
@@ -652,10 +712,6 @@ function plik_oneshot_upload(fileServerURL::String, filename::String, data::Vect
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
""" plik_oneshot_upload(fileServerURL::String, filepath::String)
|
||||
|
||||
Upload a single file to a plik server using one-shot mode.
|
||||
@@ -693,7 +749,7 @@ status, uploadid, fileid, url = plik_oneshot_upload(fileServerURL, filepath)
|
||||
# To download the uploaded file later (via curl as example):
|
||||
curl -L -O "url"
|
||||
```
|
||||
""" #[x]
|
||||
"""
|
||||
function plik_oneshot_upload(fileServerURL::String, filepath::String)
|
||||
|
||||
# ----------------------------------------- get upload id ---------------------------------------- #
|
||||
@@ -739,30 +795,4 @@ function plik_oneshot_upload(fileServerURL::String, filepath::String)
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module
|
||||
Reference in New Issue
Block a user