From 46fdf668c6ceeb6824070c7c3dc1db1770effd7f Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 19:18:12 +0700 Subject: [PATCH 01/35] update --- README.md | 16 ++- docs/architecture.md | 11 ++ docs/implementation.md | 21 ++- examples/tutorial.md | 26 +++- examples/walkthrough.md | 26 ++++ src/NATSBridge.jl | 189 +++++++++++++------------ src/NATSBridge.js | 21 ++- src/nats_bridge.py | 21 +-- test/test_js_dict_sender.js | 5 +- test/test_js_file_sender.js | 5 +- test/test_js_mix_payload_sender.js | 5 +- test/test_js_table_sender.js | 5 +- test/test_js_text_sender.js | 5 +- test/test_julia_dict_sender.jl | 5 +- test/test_julia_file_sender.jl | 5 +- test/test_julia_mix_payloads_sender.jl | 5 +- test/test_julia_table_sender.jl | 5 +- test/test_julia_text_sender.jl | 5 +- test/test_micropython_dict_sender.py | 5 +- test/test_micropython_file_sender.py | 5 +- test/test_micropython_mixed_sender.py | 5 +- test/test_micropython_text_sender.py | 5 +- 22 files changed, 260 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index bef2f2d..24b122f 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,7 @@ const env = await smartsend( ```julia using NATSBridge -env = NATSBridge.smartsend( +env, msg_json_str = NATSBridge.smartsend( subject; # NATS subject data::AbstractArray{Tuple{String, Any, String}}; # List of (dataname, data, type) nats_url::String = "nats://localhost:4222", @@ -371,6 +371,9 @@ env = NATSBridge.smartsend( reply_to::String = "", reply_to_msg_id::String = "" ) +# Returns: (msgEnvelope_v1, JSON string) +# - env: msgEnvelope_v1 object with all envelope metadata and payloads +# - msg_json_str: JSON string representation of the envelope for publishing ``` ### smartreceive @@ -636,19 +639,24 @@ data = [("students", df, "table")] NATSBridge.smartsend("/data/analysis", data) ``` -### Example 4: Request-Response Pattern +### Example 4: Request-Response Pattern with Envelope JSON -Bi-directional communication with reply-to support. +Bi-directional communication with reply-to support. The `smartsend` function now returns both the envelope object and a JSON string that can be published directly. #### Python/Micropython (Requester) ```python from nats_bridge import smartsend -env = smartsend( +env, msg_json_str = smartsend( "/device/command", [("command", {"action": "read_sensor"}, "dictionary")], reply_to="/device/response" ) +# env: msgEnvelope_v1 object +# msg_json_str: JSON string for publishing to NATS + +# The msg_json_str can also be published directly using NATS request-reply pattern +# nc.request("/device/command", msg_json_str, reply_to="/device/response") ``` #### Python/Micropython (Responder) diff --git a/docs/architecture.md b/docs/architecture.md index 38129df..e46a9fc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -387,9 +387,20 @@ function smartsend( nats_url::String = "nats://localhost:4222", fileserverUploadHandler::Function = plik_oneshot_upload, size_threshold::Int = 1_000_000 # 1MB + is_publish::Bool = true # Whether to automatically publish to NATS ) ``` +**Return Value:** +- Returns a tuple `(env, msg_json_str)` where: + - `env::msgEnvelope_v1` - The envelope object containing all metadata and payloads + - `msg_json_str::String` - JSON string representation of the envelope for publishing + +**Options:** +- `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern. + +The envelope object can be accessed directly for programmatic use, while the JSON string can be published directly to NATS using the request-reply pattern. + **Input Format:** - `data::AbstractArray{Tuple{String, Any, String}}` - **Must be a list of (dataname, data, type) tuples**: `[("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...]` - Even for single payloads: `[(dataname1, data1, "type1")]` diff --git a/docs/implementation.md b/docs/implementation.md index 6cb1265..b2e5d97 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -146,7 +146,22 @@ All three implementations (Julia, JavaScript, Python/Micropython) follow the sam └─────────────────┘ └─────────────────┘ ``` -## Files +## smartsend Return Value + +The `smartsend` function now returns a tuple containing both the envelope object and the JSON string representation: + +```julia +env, msg_json_str = smartsend(...) +# env::msgEnvelope_v1 - The envelope object with all metadata and payloads +# msg_json_str::String - JSON string for publishing to NATS +``` + +**Options:** +- `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern. + +This enables two use cases: +1. **Programmatic envelope access**: Access envelope fields directly via the `env` object +2. **Direct JSON publishing**: Publish the JSON string directly using NATS request-reply pattern ### Julia Module: [`src/NATSBridge.jl`](../src/NATSBridge.jl) @@ -345,7 +360,9 @@ df = DataFrame( ) # Send via SmartSend - wrapped in a list (type is part of each tuple) -await SmartSend("analysis_results", [("table_data", df, "table")]); +env, msg_json_str = SmartSend("analysis_results", [("table_data", df, "table")]) +# env: msgEnvelope_v1 object with all metadata and payloads +# msg_json_str: JSON string representation of the envelope for publishing ``` #### JavaScript (Receiver) diff --git a/examples/tutorial.md b/examples/tutorial.md index fcc71dc..d43f905 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -107,10 +107,15 @@ python3 -m http.server 8080 --directory /tmp/fileserver ```python from nats_bridge import smartsend -# Send a text message +# Send a text message (is_publish=True by default) data = [("message", "Hello World", "text")] -env = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") print("Message sent!") + +# Or use is_publish=False to get envelope and JSON without publishing +env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222", is_publish=False) +# env: MessageEnvelope object +# msg_json_str: JSON string for publishing to NATS ``` #### JavaScript @@ -118,12 +123,19 @@ print("Message sent!") ```javascript const { smartsend } = require('./src/NATSBridge'); -// Send a text message +// Send a text message (isPublish=true by default) await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } ], { natsUrl: "nats://localhost:4222" }); console.log("Message sent!"); + +// Or use isPublish=false to get envelope and JSON without publishing +const { env, msg_json_str } = await smartsend("/chat/room1", [ + { dataname: "message", data: "Hello World", type: "text" } +], { natsUrl: "nats://localhost:4222", isPublish: false }); +// env: MessageEnvelope object +// msg_json_str: JSON string for publishing to NATS ``` #### Julia @@ -133,7 +145,9 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +# env: msgEnvelope_v1 object with all metadata and payloads +# msg_json_str: JSON string representation of the envelope for publishing println("Message sent!") ``` @@ -279,13 +293,15 @@ from nats_bridge import smartsend # Send command with reply-to data = [("command", {"action": "read_sensor"}, "dictionary")] -env = smartsend( +env, msg_json_str = smartsend( "/device/command", data, nats_url="nats://localhost:4222", reply_to="/device/response", reply_to_msg_id="cmd-001" ) +# env: msgEnvelope_v1 object +# msg_json_str: JSON string for publishing to NATS ``` #### JavaScript (Responder) diff --git a/examples/walkthrough.md b/examples/walkthrough.md index 0e42bbb..304b88f 100644 --- a/examples/walkthrough.md +++ b/examples/walkthrough.md @@ -514,6 +514,7 @@ class SensorSender: data = [("reading", reading.to_dict(), "dictionary")] + # Default: is_publish=True (automatically publishes to NATS) smartsend( f"/sensors/{sensor_id}", data, @@ -521,6 +522,31 @@ class SensorSender: fileserver_url=self.fileserver_url ) + def prepare_message_only(self, sensor_id: str, value: float, unit: str): + """Prepare a message without publishing (is_publish=False).""" + reading = SensorReading( + sensor_id=sensor_id, + timestamp=datetime.now().isoformat(), + value=value, + unit=unit + ) + + data = [("reading", reading.to_dict(), "dictionary")] + + # With is_publish=False, returns (envelope, json_str) without publishing + env, msg_json_str = smartsend( + f"/sensors/{sensor_id}/prepare", + data, + nats_url=self.nats_url, + fileserver_url=self.fileserver_url, + is_publish=False + ) + + # Now you can publish manually using NATS request-reply pattern + # nc.request(subject, msg_json_str, reply_to=reply_to_topic) + + return env, msg_json_str + def send_batch(self, readings: List[SensorReading]): batch = SensorBatch() for reading in readings: diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 9e224ec..42185ad 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -286,35 +286,35 @@ function envelope_to_json(env::msgEnvelope_v1) # Convert payloads to JSON array if !isempty(env.payloads) - payloads_json = [] - for payload in env.payloads - payload_obj = Dict{String, Any}( - "id" => payload.id, - "dataname" => payload.dataname, - "type" => payload.type, - "transport" => payload.transport, - "encoding" => payload.encoding, - "size" => payload.size, - ) - # Include data based on transport type - if payload.transport == "direct" && payload.data !== nothing - if payload.encoding == "base64" || payload.encoding == "json" - payload_obj["data"] = payload.data - else - # For other encodings, use base64 - payload_bytes = _get_payload_bytes(payload.data) - payload_obj["data"] = Base64.base64encode(payload_bytes) - end - elseif payload.transport == "link" && payload.data !== nothing - # For link transport, data is a URL string - include directly - payload_obj["data"] = payload.data - end - if !isempty(payload.metadata) - payload_obj["metadata"] = Dict(String(k) => v for (k, v) in payload.metadata) + payloads_json = [] + for payload in env.payloads + payload_obj = Dict{String, Any}( + "id" => payload.id, + "dataname" => payload.dataname, + "type" => payload.type, + "transport" => payload.transport, + "encoding" => payload.encoding, + "size" => payload.size, + ) + # Include data based on transport type + if payload.transport == "direct" && payload.data !== nothing + if payload.encoding == "base64" || payload.encoding == "json" + payload_obj["data"] = payload.data + else + # For other encodings, use base64 + payload_bytes = _get_payload_bytes(payload.data) + payload_obj["data"] = Base64.base64encode(payload_bytes) end - push!(payloads_json, payload_obj) + elseif payload.transport == "link" && payload.data !== nothing + # For link transport, data is a URL string - include directly + payload_obj["data"] = payload.data + end + if !isempty(payload.metadata) + payload_obj["metadata"] = Dict(String(k) => v for (k, v) in payload.metadata) end - obj["payloads"] = payloads_json + push!(payloads_json, payload_obj) + end + obj["payloads"] = payloads_json end JSON.json(obj) @@ -361,6 +361,7 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c 3. Compares the serialized size against `size_threshold` 4. For small payloads: encodes as Base64, constructs a "direct" msgPayload_v1 5. For large payloads: uploads to the fileserver, constructs a "link" msgPayload_v1 with the URL +6. Converts envelope to JSON string and optionally publishes to NATS # Arguments: - `subject::String` - NATS subject to publish the message to @@ -372,18 +373,22 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c # Keyword Arguments: - `nats_url::String = DEFAULT_NATS_URL` - URL of the NATS server + - `fileserver_url = DEFAULT_FILESERVER_URL` - URL of the HTTP file server for large payloads - `fileserverUploadHandler::Function = plik_oneshot_upload` - Function to handle fileserver uploads (must return Dict with "status", "uploadid", "fileid", "url" keys) - `size_threshold::Int = DEFAULT_SIZE_THRESHOLD` - Threshold in bytes separating direct vs link transport - `correlation_id::Union{String, Nothing} = nothing` - Optional correlation ID for tracing; if `nothing`, a UUID is generated - `msg_purpose::String = "chat"` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. - - `sender_name::String = "NATSBridge"` - Name of the sender + - `sender_name::String = "default"` - Name of the sender - `receiver_name::String = ""` - Name of the receiver (empty string means broadcast) - `receiver_id::String = ""` - UUID of the receiver (empty string means broadcast) - `reply_to::String = ""` - Topic to reply to (empty string if no reply expected) - `reply_to_msg_id::String = ""` - Message ID this message is replying to + - `is_publish::Bool = true` - Whether to automatically publish the message to NATS # Return: - - A `msgEnvelope_v1` object containing metadata and transport information + - A tuple `(env, msg_json_str)` where: + - `env::msgEnvelope_v1` - The envelope object containing all metadata and payloads + - `msg_json_str::String` - JSON string representation of the envelope for publishing # Example ```jldoctest @@ -391,39 +396,43 @@ using UUIDs # Send a single payload (still wrapped in a list) data = Dict("key" => "value") -env = smartsend("my.subject", [("dataname1", data, "dictionary")]) +env, msg_json = smartsend("my.subject", [("dataname1", data, "dictionary")]) # Send multiple payloads in one message with different types data1 = Dict("key1" => "value1") data2 = rand(10_000) # Small array -env = smartsend("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")]) +env, msg_json = smartsend("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")]) # Send a large array using fileserver upload data = rand(10_000_000) # ~80 MB -env = smartsend("large.data", [("large_table", data, "table")]) +env, msg_json = smartsend("large.data", [("large_table", data, "table")]) # Mixed content (e.g., chat with text and image) -env = smartsend("chat.subject", [ +env, msg_json = smartsend("chat.subject", [ ("message_text", "Hello!", "text"), ("user_image", image_data, "image"), ("audio_clip", audio_data, "audio") ]) + +# Publish the JSON string directly using NATS request-reply pattern +# reply = NATS.request(nats_url, subject, msg_json_str; reply_to=reply_to_topic) ``` -""" +""" #[PENDING] function smartsend( subject::String, # smartreceive's subject - data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples + data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples. Use Tuple{String, Any, String}[] for empty payloads nats_url::String = DEFAULT_NATS_URL, fileserver_url = DEFAULT_FILESERVER_URL, fileserverUploadHandler::Function=plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver size_threshold::Int = DEFAULT_SIZE_THRESHOLD, correlation_id::Union{String, Nothing} = nothing, msg_purpose::String = "chat", - sender_name::String = "NATSBridge", + sender_name::String = "default", receiver_name::String = "", receiver_id::String = "", reply_to::String = "", - reply_to_msg_id::String = "" + reply_to_msg_id::String = "", + is_publish::Bool = true # some time the user want to get env and msg_json_str from this function without publishing the msg ) where {T1<:Any} # Generate correlation ID if not provided @@ -437,59 +446,59 @@ function smartsend( # Process each payload in the list payloads = msgPayload_v1[] for (dataname, payload_data, payload_type) in data - # Serialize data based on type - payload_bytes = _serialize_data(payload_data, payload_type) + # Serialize data based on type + payload_bytes = _serialize_data(payload_data, payload_type) + + payload_size = length(payload_bytes) # Calculate payload size in bytes + log_trace(cid, "Serialized payload '$dataname' (type: $payload_type) size: $payload_size bytes") # Log payload size + + # Decision: Direct vs Link + if payload_size < size_threshold # Check if payload is small enough for direct transport + # Direct path - Base64 encode and send via NATS + payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string + log_trace(cid, "Using direct transport for $payload_size bytes") # Log transport choice - payload_size = length(payload_bytes) # Calculate payload size in bytes - log_trace(cid, "Serialized payload '$dataname' (type: $payload_type) size: $payload_size bytes") # Log payload size + # Create msgPayload_v1 for direct transport + payload = msgPayload_v1( + payload_b64, + payload_type; + id = string(uuid4()), + dataname = dataname, + transport = "direct", + encoding = "base64", + size = payload_size, + metadata = Dict{String, Any}("payload_bytes" => payload_size) + ) + push!(payloads, payload) + else + # Link path - Upload to HTTP server, send URL via NATS + log_trace(cid, "Using link transport, uploading to fileserver") # Log link transport choice - # Decision: Direct vs Link - if payload_size < size_threshold # Check if payload is small enough for direct transport - # Direct path - Base64 encode and send via NATS - payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string - log_trace(cid, "Using direct transport for $payload_size bytes") # Log transport choice - - # Create msgPayload_v1 for direct transport - payload = msgPayload_v1( - payload_b64, - payload_type; - id = string(uuid4()), - dataname = dataname, - transport = "direct", - encoding = "base64", - size = payload_size, - metadata = Dict{String, Any}("payload_bytes" => payload_size) - ) - push!(payloads, payload) - else - # Link path - Upload to HTTP server, send URL via NATS - log_trace(cid, "Using link transport, uploading to fileserver") # Log link transport choice - - # Upload to HTTP server - response = fileserverUploadHandler(fileserver_url, dataname, payload_bytes) - - if response["status"] != 200 # Check if upload was successful - error("Failed to upload data to fileserver: $(response["status"])") # Throw error if upload failed - end - - url = response["url"] # URL for the uploaded data - log_trace(cid, "Uploaded to URL: $url") # Log successful upload - - # Create msgPayload_v1 for link transport - payload = msgPayload_v1( - url, - payload_type; - id = string(uuid4()), - dataname = dataname, - transport = "link", - encoding = "none", - size = payload_size, - metadata = Dict{String, Any}() - ) - push!(payloads, payload) + # Upload to HTTP server + response = fileserverUploadHandler(fileserver_url, dataname, payload_bytes) + + if response["status"] != 200 # Check if upload was successful + error("Failed to upload data to fileserver: $(response["status"])") # Throw error if upload failed end + + url = response["url"] # URL for the uploaded data + log_trace(cid, "Uploaded to URL: $url") # Log successful upload + + # Create msgPayload_v1 for link transport + payload = msgPayload_v1( + url, + payload_type; + id = string(uuid4()), + dataname = dataname, + transport = "link", + encoding = "none", + size = payload_size, + metadata = Dict{String, Any}() + ) + push!(payloads, payload) + end end - + # Create msgEnvelope_v1 with all payloads env = msgEnvelope_v1( subject, @@ -507,10 +516,12 @@ function smartsend( metadata = Dict{String, Any}(), ) - msg_json = envelope_to_json(env) # Convert envelope to JSON - publish_message(nats_url, subject, msg_json, cid) # Publish message to NATS + msg_json_str = envelope_to_json(env) # Convert envelope to JSON + if is_publish + publish_message(nats_url, subject, msg_json_str, cid) # Publish message to NATS + end - return env # Return the envelope for tracking + return (env, msg_json_str) end diff --git a/src/NATSBridge.js b/src/NATSBridge.js index f939433..c45f5e8 100644 --- a/src/NATSBridge.js +++ b/src/NATSBridge.js @@ -460,8 +460,9 @@ async function smartsend(subject, data, options = {}) { * @param {string} options.receiverId - UUID of the receiver (default: "") * @param {string} options.replyTo - Topic to reply to (default: "") * @param {string} options.replyToMsgId - Message ID this message is replying to (default: "") + * @param {boolean} options.isPublish - Whether to automatically publish the message to NATS (default: true) * - * @returns {Promise} - The envelope for tracking + * @returns {Promise} - An object with { env: MessageEnvelope, msg_json_str: string } */ const { natsUrl = DEFAULT_NATS_URL, @@ -474,7 +475,8 @@ async function smartsend(subject, data, options = {}) { receiverName = "", receiverId = "", replyTo = "", - replyToMsgId = "" + replyToMsgId = "", + isPublish = true // Whether to automatically publish the message to NATS } = options; log_trace(correlationId, `Starting smartsend for subject: ${subject}`); @@ -556,10 +558,19 @@ async function smartsend(subject, data, options = {}) { payloads: payloads }); - // Publish message to NATS - await publish_message(natsUrl, subject, env.toString(), correlationId); + // Convert envelope to JSON string + const msg_json_str = env.toString(); - return env; + // Publish to NATS if isPublish is true + if (isPublish) { + await publish_message(natsUrl, subject, msg_json_str, correlationId); + } + + // Return both envelope and JSON string (tuple-like structure) + return { + env: env, + msg_json_str: msg_json_str + }; } // Helper: Publish message to NATS diff --git a/src/nats_bridge.py b/src/nats_bridge.py index a0fdcc3..63ebedc 100644 --- a/src/nats_bridge.py +++ b/src/nats_bridge.py @@ -437,7 +437,7 @@ def plik_oneshot_upload(file_server_url, filename, data): def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_FILESERVER_URL, fileserver_upload_handler=plik_oneshot_upload, size_threshold=DEFAULT_SIZE_THRESHOLD, correlation_id=None, msg_purpose="chat", sender_name="NATSBridge", - receiver_name="", receiver_id="", reply_to="", reply_to_msg_id=""): + receiver_name="", receiver_id="", reply_to="", reply_to_msg_id="", is_publish=True): """Send data either directly via NATS or via a fileserver URL, depending on payload size. This function intelligently routes data delivery based on payload size relative to a threshold. @@ -459,9 +459,12 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F receiver_id: UUID of the receiver reply_to: Topic to reply to reply_to_msg_id: Message ID this message is replying to + is_publish: Whether to automatically publish the message to NATS (default: True) Returns: - MessageEnvelope: The envelope object for tracking + tuple: (env, msg_json_str) where: + - env: MessageEnvelope object with all metadata and payloads + - msg_json_str: JSON string representation of the envelope for publishing """ # Generate correlation ID if not provided cid = correlation_id if correlation_id else str(uuid.uuid4()) @@ -549,13 +552,15 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F msg_json = env.to_json() - # Publish to NATS - nats_conn = NATSConnection(nats_url) - nats_conn.connect() - nats_conn.publish(subject, msg_json) - nats_conn.close() + # Publish to NATS if is_publish is True + if is_publish: + nats_conn = NATSConnection(nats_url) + nats_conn.connect() + nats_conn.publish(subject, msg_json) + nats_conn.close() - return env + # Return tuple of (envelope, json_string) for both direct and link transport + return (env, msg_json) def smartreceive(msg, fileserver_download_handler=_fetch_with_backoff, max_retries=5, diff --git a/test/test_js_dict_sender.js b/test/test_js_dict_sender.js index 4eaf7e8..99b0912 100644 --- a/test/test_js_dict_sender.js +++ b/test/test_js_dict_sender.js @@ -118,7 +118,7 @@ async function test_dict_send() { // Use smartsend with dictionary type // For small Dictionary: will use direct transport (JSON encoded) // For large Dictionary: will use link transport (uploaded to fileserver) - const env = await smartsend( + const { env, msg_json_str } = await smartsend( SUBJECT, [data1, data2], { @@ -132,7 +132,8 @@ async function test_dict_send() { receiverName: "", receiverId: "", replyTo: "", - replyToMsgId: "" + replyToMsgId: "", + isPublish: true // Publish the message to NATS } ); diff --git a/test/test_js_file_sender.js b/test/test_js_file_sender.js index a53986e..80b7742 100644 --- a/test/test_js_file_sender.js +++ b/test/test_js_file_sender.js @@ -98,7 +98,7 @@ async function test_large_binary_send() { // Use smartsend with binary type - will automatically use link transport // if file size exceeds the threshold (1MB by default) - const env = await smartsend( + const { env, msg_json_str } = await smartsend( SUBJECT, [data1, data2], { @@ -112,7 +112,8 @@ async function test_large_binary_send() { receiverName: "", receiverId: "", replyTo: "", - replyToMsgId: "" + replyToMsgId: "", + isPublish: true // Publish the message to NATS } ); diff --git a/test/test_js_mix_payload_sender.js b/test/test_js_mix_payload_sender.js index f16389c..e25eeb3 100644 --- a/test/test_js_mix_payload_sender.js +++ b/test/test_js_mix_payload_sender.js @@ -222,7 +222,7 @@ async function test_mix_send() { ]; // Use smartsend with mixed content - const env = await smartsend( + const { env, msg_json_str } = await smartsend( SUBJECT, payloads, { @@ -236,7 +236,8 @@ async function test_mix_send() { receiverName: "", receiverId: "", replyTo: "", - replyToMsgId: "" + replyToMsgId: "", + isPublish: true // Publish the message to NATS } ); diff --git a/test/test_js_table_sender.js b/test/test_js_table_sender.js index 4b5a8a9..2aedabd 100644 --- a/test/test_js_table_sender.js +++ b/test/test_js_table_sender.js @@ -118,7 +118,7 @@ async function test_table_send() { // Use smartsend with table type // For small Table: will use direct transport (Arrow IPC encoded) // For large Table: will use link transport (uploaded to fileserver) - const env = await smartsend( + const { env, msg_json_str } = await smartsend( SUBJECT, [data1, data2], { @@ -132,7 +132,8 @@ async function test_table_send() { receiverName: "", receiverId: "", replyTo: "", - replyToMsgId: "" + replyToMsgId: "", + isPublish: true // Publish the message to NATS } ); diff --git a/test/test_js_text_sender.js b/test/test_js_text_sender.js index 6a75f59..81088cc 100644 --- a/test/test_js_text_sender.js +++ b/test/test_js_text_sender.js @@ -94,7 +94,7 @@ async function test_text_send() { // Use smartsend with text type // For small text: will use direct transport (Base64 encoded UTF-8) // For large text: will use link transport (uploaded to fileserver) - const env = await smartsend( + const { env, msg_json_str } = await smartsend( SUBJECT, [data1, data2], { @@ -108,7 +108,8 @@ async function test_text_send() { receiverName: "", receiverId: "", replyTo: "", - replyToMsgId: "" + replyToMsgId: "", + isPublish: true // Publish the message to NATS } ); diff --git a/test/test_julia_dict_sender.jl b/test/test_julia_dict_sender.jl index 9c28d5c..75e6aff 100644 --- a/test/test_julia_dict_sender.jl +++ b/test/test_julia_dict_sender.jl @@ -92,7 +92,7 @@ function test_dict_send() # Use smartsend with dictionary type # For small Dictionary: will use direct transport (JSON encoded) # For large Dictionary: will use link transport (uploaded to fileserver) - env = NATSBridge.smartsend( + env, msg_json = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL, @@ -105,7 +105,8 @@ function test_dict_send() receiver_name = "", receiver_id = "", reply_to = "", - reply_to_msg_id = "" + reply_to_msg_id = "", + is_publish = true # Publish the message to NATS ) log_trace("Sent message with $(length(env.payloads)) payloads") diff --git a/test/test_julia_file_sender.jl b/test/test_julia_file_sender.jl index 64b83ff..a4b086f 100644 --- a/test/test_julia_file_sender.jl +++ b/test/test_julia_file_sender.jl @@ -79,7 +79,7 @@ function test_large_binary_send() # Use smartsend with binary type - will automatically use link transport # if file size exceeds the threshold (1MB by default) # API: smartsend(subject, [(dataname, data, type), ...]; keywords...) - env = NATSBridge.smartsend( + env, msg_json = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL; @@ -92,7 +92,8 @@ function test_large_binary_send() receiver_name = "", receiver_id = "", reply_to = "", - reply_to_msg_id = "" + reply_to_msg_id = "", + is_publish = true # Publish the message to NATS ) log_trace("Sent message with transport: $(env.payloads[1].transport)") diff --git a/test/test_julia_mix_payloads_sender.jl b/test/test_julia_mix_payloads_sender.jl index c9fab9b..c1805b7 100644 --- a/test/test_julia_mix_payloads_sender.jl +++ b/test/test_julia_mix_payloads_sender.jl @@ -186,7 +186,7 @@ function test_mix_send() ] # Use smartsend with mixed content - env = NATSBridge.smartsend( + env, msg_json = NATSBridge.smartsend( SUBJECT, payloads; # List of (dataname, data, type) tuples nats_url = NATS_URL, @@ -199,7 +199,8 @@ function test_mix_send() receiver_name = "", receiver_id = "", reply_to = "", - reply_to_msg_id = "" + reply_to_msg_id = "", + is_publish = true # Publish the message to NATS ) log_trace("Sent message with $(length(env.payloads)) payloads") diff --git a/test/test_julia_table_sender.jl b/test/test_julia_table_sender.jl index ed8c4b7..338e73f 100644 --- a/test/test_julia_table_sender.jl +++ b/test/test_julia_table_sender.jl @@ -90,7 +90,7 @@ function test_table_send() # Use smartsend with table type # For small DataFrame: will use direct transport (Base64 encoded Arrow IPC) # For large DataFrame: will use link transport (uploaded to fileserver) - env = NATSBridge.smartsend( + env, msg_json = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL, @@ -103,7 +103,8 @@ function test_table_send() receiver_name = "", receiver_id = "", reply_to = "", - reply_to_msg_id = "" + reply_to_msg_id = "", + is_publish = true # Publish the message to NATS ) log_trace("Sent message with $(length(env.payloads)) payloads") diff --git a/test/test_julia_text_sender.jl b/test/test_julia_text_sender.jl index 782a12b..4746f82 100644 --- a/test/test_julia_text_sender.jl +++ b/test/test_julia_text_sender.jl @@ -75,7 +75,7 @@ function test_text_send() # Use smartsend with text type # For small text: will use direct transport (Base64 encoded UTF-8) # For large text: will use link transport (uploaded to fileserver) - env = NATSBridge.smartsend( + env, msg_json = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL, @@ -88,7 +88,8 @@ function test_text_send() receiver_name = "", receiver_id = "", reply_to = "", - reply_to_msg_id = "" + reply_to_msg_id = "", + is_publish = true # Publish the message to NATS ) log_trace("Sent message with $(length(env.payloads)) payloads") diff --git a/test/test_micropython_dict_sender.py b/test/test_micropython_dict_sender.py index 3d63106..abdf197 100644 --- a/test/test_micropython_dict_sender.py +++ b/test/test_micropython_dict_sender.py @@ -64,7 +64,7 @@ def main(): log_trace(correlation_id, f"Correlation ID: {correlation_id}") # Use smartsend with dictionary type - env = smartsend( + env, msg_json = smartsend( SUBJECT, [data1, data2], # List of (dataname, data, type) tuples nats_url=NATS_URL, @@ -76,7 +76,8 @@ def main(): receiver_name="", receiver_id="", reply_to="", - reply_to_msg_id="" + reply_to_msg_id="", + is_publish=True # Publish the message to NATS ) log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") diff --git a/test/test_micropython_file_sender.py b/test/test_micropython_file_sender.py index 9219c0f..dc06014 100644 --- a/test/test_micropython_file_sender.py +++ b/test/test_micropython_file_sender.py @@ -44,7 +44,7 @@ def main(): log_trace(correlation_id, f"Correlation ID: {correlation_id}") # Use smartsend with binary type - env = smartsend( + env, msg_json = smartsend( SUBJECT, [data1, data2], # List of (dataname, data, type) tuples nats_url=NATS_URL, @@ -56,7 +56,8 @@ def main(): receiver_name="", receiver_id="", reply_to="", - reply_to_msg_id="" + reply_to_msg_id="", + is_publish=True # Publish the message to NATS ) log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") diff --git a/test/test_micropython_mixed_sender.py b/test/test_micropython_mixed_sender.py index 00e24d3..437087b 100644 --- a/test/test_micropython_mixed_sender.py +++ b/test/test_micropython_mixed_sender.py @@ -58,7 +58,7 @@ def main(): log_trace(correlation_id, f"Correlation ID: {correlation_id}") # Use smartsend with mixed types - env = smartsend( + env, msg_json = smartsend( SUBJECT, data, # List of (dataname, data, type) tuples nats_url=NATS_URL, @@ -70,7 +70,8 @@ def main(): receiver_name="", receiver_id="", reply_to="", - reply_to_msg_id="" + reply_to_msg_id="", + is_publish=True # Publish the message to NATS ) log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") diff --git a/test/test_micropython_text_sender.py b/test/test_micropython_text_sender.py index 26200ed..c048216 100644 --- a/test/test_micropython_text_sender.py +++ b/test/test_micropython_text_sender.py @@ -46,7 +46,7 @@ def main(): # Use smartsend with text type # For small text: will use direct transport (Base64 encoded UTF-8) # For large text: will use link transport (uploaded to fileserver) - env = smartsend( + env, msg_json = smartsend( SUBJECT, [data1, data2], # List of (dataname, data, type) tuples nats_url=NATS_URL, @@ -58,7 +58,8 @@ def main(): receiver_name="", receiver_id="", reply_to="", - reply_to_msg_id="" + reply_to_msg_id="", + is_publish=True # Publish the message to NATS ) log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") -- 2.49.1 From 0c2cca30ed8ec3d60c5c4fe80d6518fb09f728b9 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 20:34:08 +0700 Subject: [PATCH 02/35] update --- README.md | 15 +++++++++------ src/NATSBridge.jl | 8 ++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 24b122f..ca5bc44 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ from nats_bridge import smartsend # Send a text message data = [("message", "Hello World", "text")] -env = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") print("Message sent!") ``` @@ -183,7 +183,7 @@ print("Message sent!") const { smartsend } = require('./src/NATSBridge'); // Send a text message -await smartsend("/chat/room1", [ +const { env, msg_json_str } = await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } ], { natsUrl: "nats://localhost:4222" }); @@ -197,7 +197,7 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env = NATSBridge.smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, msg_json_str = NATSBridge.smartsend("/chat/room1", data, nats_url="nats://localhost:4222") println("Message sent!") ``` @@ -323,7 +323,8 @@ env = smartsend( receiver_name="", # Receiver name (empty = broadcast) receiver_id="", # Receiver UUID (empty = broadcast) reply_to="", # Reply topic - reply_to_msg_id="" # Reply message ID + reply_to_msg_id="", # Reply message ID + is_publish=True # Whether to automatically publish to NATS ) ``` @@ -346,7 +347,8 @@ const env = await smartsend( receiverName: "", receiverId: "", replyTo: "", - replyToMsgId: "" + replyToMsgId: "", + isPublish: true // Whether to automatically publish to NATS } ); ``` @@ -369,7 +371,8 @@ env, msg_json_str = NATSBridge.smartsend( receiver_name::String = "", receiver_id::String = "", reply_to::String = "", - reply_to_msg_id::String = "" + reply_to_msg_id::String = "", + is_publish::Bool = true # Whether to automatically publish to NATS ) # Returns: (msgEnvelope_v1, JSON string) # - env: msgEnvelope_v1 object with all envelope metadata and payloads diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 42185ad..d5aae71 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -377,9 +377,9 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c - `fileserverUploadHandler::Function = plik_oneshot_upload` - Function to handle fileserver uploads (must return Dict with "status", "uploadid", "fileid", "url" keys) - `size_threshold::Int = DEFAULT_SIZE_THRESHOLD` - Threshold in bytes separating direct vs link transport - `correlation_id::Union{String, Nothing} = nothing` - Optional correlation ID for tracing; if `nothing`, a UUID is generated - - `msg_purpose::String = "chat"` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. - - `sender_name::String = "default"` - Name of the sender - - `receiver_name::String = ""` - Name of the receiver (empty string means broadcast) + - `msg_purpose::String = "chat"` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. + - `sender_name::String = "NATSBridge"` - Name of the sender + - `receiver_name::String = ""` - Name of the receiver (empty string means broadcast) - `receiver_id::String = ""` - UUID of the receiver (empty string means broadcast) - `reply_to::String = ""` - Topic to reply to (empty string if no reply expected) - `reply_to_msg_id::String = ""` - Message ID this message is replying to @@ -427,7 +427,7 @@ function smartsend( size_threshold::Int = DEFAULT_SIZE_THRESHOLD, correlation_id::Union{String, Nothing} = nothing, msg_purpose::String = "chat", - sender_name::String = "default", + sender_name::String = "NATSBridge", receiver_name::String = "", receiver_id::String = "", reply_to::String = "", -- 2.49.1 From 263508b8f74366931601f3a26b5bab22e1388539 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 20:50:41 +0700 Subject: [PATCH 03/35] update --- Project.toml | 2 +- examples/tutorial.md | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Project.toml b/Project.toml index e362d04..7dcfae6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "NATSBridge" uuid = "f2724d33-f338-4a57-b9f8-1be882570d10" -version = "0.4.2" +version = "0.4.3" authors = ["narawat "] [deps] diff --git a/examples/tutorial.md b/examples/tutorial.md index d43f905..8c874c4 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -208,7 +208,7 @@ config = { # Send as dictionary type data = [("config", config, "dictionary")] -env = smartsend("/device/config", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/device/config", data, nats_url="nats://localhost:4222") ``` #### JavaScript @@ -222,7 +222,7 @@ const config = { update_interval: 60 }; -await smartsend("/device/config", [ +const { env, msg_json_str } = await smartsend("/device/config", [ { dataname: "config", data: config, type: "dictionary" } ]); ``` @@ -239,7 +239,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -smartsend("/device/config", data) +env, msg_json_str = smartsend("/device/config", data) ``` ### Example 2: Sending Binary Data (Image) @@ -255,7 +255,7 @@ with open("image.png", "rb") as f: # Send as binary type data = [("user_image", image_data, "binary")] -env = smartsend("/chat/image", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/chat/image", data, nats_url="nats://localhost:4222") ``` #### JavaScript @@ -267,7 +267,7 @@ const { smartsend } = require('./src/NATSBridge'); const fs = require('fs'); const image_data = fs.readFileSync('image.png'); -await smartsend("/chat/image", [ +const { env, msg_json_str } = await smartsend("/chat/image", [ { dataname: "user_image", data: image_data, type: "binary" } ]); ``` @@ -281,7 +281,7 @@ using NATSBridge image_data = read("image.png") data = [("user_image", image_data, "binary")] -smartsend("/chat/image", data) +env, msg_json_str = smartsend("/chat/image", data) ``` ### Example 3: Request-Response Pattern @@ -358,7 +358,7 @@ import os large_data = os.urandom(2_000_000) # 2MB of random data # Send with file server URL -env = smartsend( +env, msg_json_str = smartsend( "/data/large", [("large_file", large_data, "binary")], nats_url="nats://localhost:4222", @@ -380,7 +380,7 @@ const largeData = new ArrayBuffer(2_000_000); const view = new Uint8Array(largeData); view.fill(42); // Fill with some data -await smartsend("/data/large", [ +const { env, msg_json_str } = await smartsend("/data/large", [ { dataname: "large_file", data: largeData, type: "binary" } ], { fileserverUrl: "http://localhost:8080", @@ -396,7 +396,7 @@ using NATSBridge # Create large data (> 1MB) large_data = rand(UInt8, 2_000_000) -env = smartsend( +env, msg_json_str = smartsend( "/data/large", [("large_file", large_data, "binary")], fileserver_url="http://localhost:8080" @@ -425,7 +425,7 @@ data = [ ("user_avatar", image_data, "image") ] -env = smartsend("/chat/mixed", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/chat/mixed", data, nats_url="nats://localhost:4222") ``` #### JavaScript @@ -435,7 +435,7 @@ const { smartsend } = require('./src/NATSBridge'); const fs = require('fs'); -await smartsend("/chat/mixed", [ +const { env, msg_json_str } = await smartsend("/chat/mixed", [ { dataname: "message_text", data: "Hello with image!", @@ -461,7 +461,7 @@ data = [ ("user_avatar", image_data, "image") ] -smartsend("/chat/mixed", data) +env, msg_json_str = smartsend("/chat/mixed", data) ``` ### Example 6: Table Data (Arrow IPC) @@ -483,7 +483,7 @@ df = pd.DataFrame({ # Send as table type data = [("students", df, "table")] -env = smartsend("/data/students", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/data/students", data, nats_url="nats://localhost:4222") ``` #### Julia @@ -500,7 +500,7 @@ df = DataFrame( ) data = [("students", df, "table")] -smartsend("/data/students", data) +env, msg_json_str = smartsend("/data/students", data) ``` --- @@ -519,7 +519,7 @@ using NATSBridge # Send dictionary from Julia to JavaScript config = Dict("step_size" => 0.01, "iterations" => 1000) data = [("config", config, "dictionary")] -smartsend("/analysis/config", data, nats_url="nats://localhost:4222") +env, msg_json_str = smartsend("/analysis/config", data, nats_url="nats://localhost:4222") ``` #### JavaScript Receiver @@ -544,7 +544,7 @@ for (const payload of envelope.payloads) { ```javascript const { smartsend } = require('./src/NATSBridge'); -await smartsend("/data/transfer", [ +const { env, msg_json_str } = await smartsend("/data/transfer", [ { dataname: "message", data: "Hello from JS!", type: "text" } ]); ``` @@ -568,7 +568,7 @@ for dataname, data, type in envelope["payloads"]: from nats_bridge import smartsend data = [("message", "Hello from Python!", "text")] -smartsend("/chat/python", data) +env, msg_json_str = smartsend("/chat/python", data) ``` #### Julia Receiver -- 2.49.1 From d99dc41be94c02663d32f577339e379893094219 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 21:09:36 +0700 Subject: [PATCH 04/35] update --- examples/walkthrough.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/walkthrough.md b/examples/walkthrough.md index 304b88f..5bcb566 100644 --- a/examples/walkthrough.md +++ b/examples/walkthrough.md @@ -132,7 +132,7 @@ class ChatUI { }); } - await smartsend( + const { env, msg_json_str } = await smartsend( `/chat/${this.currentRoom}`, data, { @@ -304,7 +304,7 @@ class FileUploadService { type: 'binary' }]; - const envelope = await smartsend( + const { env, msg_json_str } = await smartsend( `/files/${recipient}`, data, { @@ -314,7 +314,7 @@ class FileUploadService { } ); - return envelope; + return env; } async uploadLargeFile(filePath, recipient) { @@ -833,7 +833,7 @@ class DashboardUI { async refreshData() { // Request fresh data - await smartsend("/dashboard/request", [ + const { env, msg_json_str } = await smartsend("/dashboard/request", [ { dataname: "request", data: { type: "refresh" }, type: "dictionary" } ], { fileserverUrl: window.config.fileserver_url -- 2.49.1 From 76f42be7405c9556722da60e6d8b8a0016c7e36e Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 21:32:22 +0700 Subject: [PATCH 05/35] update --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ca5bc44..0ddf7fa 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env, msg_json_str = NATSBridge.smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, msg_json_str = NATSBridge.smartsend("/chat/room1", data; nats_url="nats://localhost:4222") println("Message sent!") ``` @@ -310,7 +310,7 @@ Sends data either directly via NATS or via a fileserver URL, depending on payloa ```python from nats_bridge import smartsend -env = smartsend( +env, msg_json_str = smartsend( subject, # NATS subject to publish to data, # List of (dataname, data, type) tuples nats_url="nats://localhost:4222", # NATS server URL @@ -333,7 +333,7 @@ env = smartsend( ```javascript const { smartsend } = require('./src/NATSBridge'); -const env = await smartsend( +const { env, msg_json_str } = await smartsend( subject, // NATS subject data, // Array of {dataname, data, type} { @@ -359,7 +359,7 @@ const env = await smartsend( using NATSBridge env, msg_json_str = NATSBridge.smartsend( - subject; # NATS subject + subject, # NATS subject data::AbstractArray{Tuple{String, Any, String}}; # List of (dataname, data, type) nats_url::String = "nats://localhost:4222", fileserver_url = "http://localhost:8080", @@ -465,7 +465,7 @@ smartsend("/topic", data) ```javascript await smartsend("/topic", [ { dataname: "message", data: "Hello", type: "text" } -]); +], { isPublish: false }); ``` #### Julia @@ -488,13 +488,13 @@ smartsend("/topic", data, fileserver_url="http://localhost:8080") ```javascript await smartsend("/topic", [ { dataname: "file", data: largeData, type: "binary" } -], { fileserverUrl: "http://localhost:8080" }); +], { fileserverUrl: "http://localhost:8080", isPublish: false }); ``` #### Julia ```julia data = [("file", large_data, "binary")] -smartsend("/topic", data, fileserver_url="http://localhost:8080") +smartsend("/topic", data; fileserver_url="http://localhost:8080") ``` --- @@ -517,14 +517,14 @@ data = [ ("large_document", large_file_data, "binary") ] -smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") +env, msg_json_str = smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") ``` #### JavaScript ```javascript const { smartsend } = require('./src/NATSBridge'); -await smartsend("/chat/room1", [ +const { env, msg_json_str } = await smartsend("/chat/room1", [ { dataname: "message_text", data: "Hello!", type: "text" }, { dataname: "user_avatar", data: image_data, type: "image" }, { dataname: "large_document", data: large_file_data, type: "binary" } @@ -543,7 +543,7 @@ data = [ ("large_document", large_file_data, "binary") ] -NATSBridge.smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") +env, msg_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") ``` ### Example 2: Dictionary Exchange @@ -561,7 +561,7 @@ config = { } data = [("config", config, "dictionary")] -smartsend("/device/config", data) +env, msg_json_str = smartsend("/device/config", data) ``` #### JavaScript @@ -574,7 +574,7 @@ const config = { update_interval: 60 }; -await smartsend("/device/config", [ +const { env, msg_json_str } = await smartsend("/device/config", [ { dataname: "config", data: config, type: "dictionary" } ]); ``` @@ -590,7 +590,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -NATSBridge.smartsend("/device/config", data) +env, msg_json_str = NATSBridge.smartsend("/device/config", data) ``` ### Example 3: Table Data (Arrow IPC) @@ -609,7 +609,7 @@ df = pd.DataFrame({ }) data = [("students", df, "table")] -smartsend("/data/analysis", data) +env, msg_json_str = smartsend("/data/analysis", data) ``` #### JavaScript @@ -622,7 +622,7 @@ const tableData = [ { id: 3, name: "Charlie", score: 92 } ]; -await smartsend("/data/analysis", [ +const { env, msg_json_str } = await smartsend("/data/analysis", [ { dataname: "students", data: tableData, type: "table" } ]); ``` @@ -639,7 +639,7 @@ df = DataFrame( ) data = [("students", df, "table")] -NATSBridge.smartsend("/data/analysis", data) +env, msg_json_str = NATSBridge.smartsend("/data/analysis", data) ``` ### Example 4: Request-Response Pattern with Envelope JSON @@ -738,7 +738,7 @@ using NATSBridge env = NATSBridge.smartsend( "/device/command", - [("command", Dict("action" => "read_sensor"), "dictionary")], + [("command", Dict("action" => "read_sensor"), "dictionary")]; reply_to="/device/response" ) ``` -- 2.49.1 From 0fa6eaf95bf041e8eaea05bc96443e109fa203a7 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 21:37:50 +0700 Subject: [PATCH 06/35] update --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0ddf7fa..b250198 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,7 @@ smartsend("/topic", data) ```javascript await smartsend("/topic", [ { dataname: "message", data: "Hello", type: "text" } -], { isPublish: false }); +]); ``` #### Julia @@ -488,7 +488,7 @@ smartsend("/topic", data, fileserver_url="http://localhost:8080") ```javascript await smartsend("/topic", [ { dataname: "file", data: largeData, type: "binary" } -], { fileserverUrl: "http://localhost:8080", isPublish: false }); +], { fileserverUrl: "http://localhost:8080" }); ``` #### Julia @@ -694,7 +694,7 @@ asyncio.run(main()) ```javascript const { smartsend } = require('./src/NATSBridge'); -await smartsend("/device/command", [ +const { env, msg_json_str } = await smartsend("/device/command", [ { dataname: "command", data: { action: "read_sensor" }, type: "dictionary" } ], { replyTo: "/device/response" @@ -736,7 +736,7 @@ main(); ```julia using NATSBridge -env = NATSBridge.smartsend( +env, msg_json_str = NATSBridge.smartsend( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; reply_to="/device/response" -- 2.49.1 From 99bf57b1541e58ec568b6c7263807333577686bb Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 21:43:09 +0700 Subject: [PATCH 07/35] update --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index b250198..c48cc8a 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ from nats_bridge import smartsend # Send a text message data = [("message", "Hello World", "text")] -env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +envelope, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") print("Message sent!") ``` @@ -183,7 +183,7 @@ print("Message sent!") const { smartsend } = require('./src/NATSBridge'); // Send a text message -const { env, msg_json_str } = await smartsend("/chat/room1", [ +const { envelope, msg_json_str } = await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } ], { natsUrl: "nats://localhost:4222" }); @@ -197,7 +197,7 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env, msg_json_str = NATSBridge.smartsend("/chat/room1", data; nats_url="nats://localhost:4222") +envelope, msg_json_str = NATSBridge.smartsend("/chat/room1", data; nats_url="nats://localhost:4222") println("Message sent!") ``` @@ -283,7 +283,7 @@ function test_receive() log_trace("Received message on $(msg.subject)") # Receive and process message - envelope = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + envelope, msg_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) for (dataname, data, type) in envelope["payloads"] println("Received $dataname: $data") end @@ -310,7 +310,7 @@ Sends data either directly via NATS or via a fileserver URL, depending on payloa ```python from nats_bridge import smartsend -env, msg_json_str = smartsend( +envelope, msg_json_str = smartsend( subject, # NATS subject to publish to data, # List of (dataname, data, type) tuples nats_url="nats://localhost:4222", # NATS server URL @@ -333,7 +333,7 @@ env, msg_json_str = smartsend( ```javascript const { smartsend } = require('./src/NATSBridge'); -const { env, msg_json_str } = await smartsend( +const { envelope, msg_json_str } = await smartsend( subject, // NATS subject data, // Array of {dataname, data, type} { @@ -358,7 +358,7 @@ const { env, msg_json_str } = await smartsend( ```julia using NATSBridge -env, msg_json_str = NATSBridge.smartsend( +envelope, msg_json_str = NATSBridge.smartsend( subject, # NATS subject data::AbstractArray{Tuple{String, Any, String}}; # List of (dataname, data, type) nats_url::String = "nats://localhost:4222", @@ -375,7 +375,7 @@ env, msg_json_str = NATSBridge.smartsend( is_publish::Bool = true # Whether to automatically publish to NATS ) # Returns: (msgEnvelope_v1, JSON string) -# - env: msgEnvelope_v1 object with all envelope metadata and payloads +# - envelope: msgEnvelope_v1 object with all envelope metadata and payloads # - msg_json_str: JSON string representation of the envelope for publishing ``` @@ -423,7 +423,7 @@ const envelope = await smartreceive( using NATSBridge # Note: msg is a NATS.Msg object passed from the subscription callback -envelope = NATSBridge.smartreceive( +envelope, msg_json_str = NATSBridge.smartreceive( msg::NATS.Msg; fileserverDownloadHandler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -517,14 +517,14 @@ data = [ ("large_document", large_file_data, "binary") ] -env, msg_json_str = smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") +envelope, msg_json_str = smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") ``` #### JavaScript ```javascript const { smartsend } = require('./src/NATSBridge'); -const { env, msg_json_str } = await smartsend("/chat/room1", [ +const { envelope, msg_json_str } = await smartsend("/chat/room1", [ { dataname: "message_text", data: "Hello!", type: "text" }, { dataname: "user_avatar", data: image_data, type: "image" }, { dataname: "large_document", data: large_file_data, type: "binary" } @@ -543,7 +543,7 @@ data = [ ("large_document", large_file_data, "binary") ] -env, msg_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") +envelope, msg_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") ``` ### Example 2: Dictionary Exchange @@ -561,7 +561,7 @@ config = { } data = [("config", config, "dictionary")] -env, msg_json_str = smartsend("/device/config", data) +envelope, msg_json_str = smartsend("/device/config", data) ``` #### JavaScript @@ -574,7 +574,7 @@ const config = { update_interval: 60 }; -const { env, msg_json_str } = await smartsend("/device/config", [ +const { envelope, msg_json_str } = await smartsend("/device/config", [ { dataname: "config", data: config, type: "dictionary" } ]); ``` @@ -590,7 +590,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -env, msg_json_str = NATSBridge.smartsend("/device/config", data) +envelope, msg_json_str = NATSBridge.smartsend("/device/config", data) ``` ### Example 3: Table Data (Arrow IPC) @@ -609,7 +609,7 @@ df = pd.DataFrame({ }) data = [("students", df, "table")] -env, msg_json_str = smartsend("/data/analysis", data) +envelope, msg_json_str = smartsend("/data/analysis", data) ``` #### JavaScript @@ -622,7 +622,7 @@ const tableData = [ { id: 3, name: "Charlie", score: 92 } ]; -const { env, msg_json_str } = await smartsend("/data/analysis", [ +const { envelope, msg_json_str } = await smartsend("/data/analysis", [ { dataname: "students", data: tableData, type: "table" } ]); ``` @@ -639,7 +639,7 @@ df = DataFrame( ) data = [("students", df, "table")] -env, msg_json_str = NATSBridge.smartsend("/data/analysis", data) +envelope, msg_json_str = NATSBridge.smartsend("/data/analysis", data) ``` ### Example 4: Request-Response Pattern with Envelope JSON @@ -650,12 +650,12 @@ Bi-directional communication with reply-to support. The `smartsend` function now ```python from nats_bridge import smartsend -env, msg_json_str = smartsend( +envelope, msg_json_str = smartsend( "/device/command", [("command", {"action": "read_sensor"}, "dictionary")], reply_to="/device/response" ) -# env: msgEnvelope_v1 object +# envelope: msgEnvelope_v1 object # msg_json_str: JSON string for publishing to NATS # The msg_json_str can also be published directly using NATS request-reply pattern @@ -694,7 +694,7 @@ asyncio.run(main()) ```javascript const { smartsend } = require('./src/NATSBridge'); -const { env, msg_json_str } = await smartsend("/device/command", [ +const { envelope, msg_json_str } = await smartsend("/device/command", [ { dataname: "command", data: { action: "read_sensor" }, type: "dictionary" } ], { replyTo: "/device/response" @@ -736,7 +736,7 @@ main(); ```julia using NATSBridge -env, msg_json_str = NATSBridge.smartsend( +envelope, msg_json_str = NATSBridge.smartsend( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; reply_to="/device/response" @@ -755,7 +755,7 @@ const NATS_URL = "nats://localhost:4222" function test_responder() conn = NATS.connect(NATS_URL) NATS.subscribe(conn, SUBJECT) do msg - envelope = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + envelope, msg_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) for (dataname, data, type) in envelope["payloads"] if dataname == "command" && data["action"] == "read_sensor" response = Dict("sensor_id" => "sensor-001", "value" => 42.5) @@ -847,7 +847,7 @@ const NATS_URL = "nats://localhost:4222" function test_receiver() conn = NATS.connect(NATS_URL) NATS.subscribe(conn, SUBJECT) do msg - envelope = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + envelope, msg_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) for (dataname, data, type) in envelope["payloads"] if dataname == "temperature" println("Temperature: $data") -- 2.49.1 From 7853e94d2e77e20b0d3cff59a56f702246c0126d Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 21:54:50 +0700 Subject: [PATCH 08/35] update --- docs/architecture.md | 4 +- docs/implementation.md | 8 ++-- examples/tutorial.md | 52 +++++++++++++------------- examples/walkthrough.md | 12 +++--- src/NATSBridge.jl | 14 +++---- src/NATSBridge.js | 8 ++-- src/nats_bridge.py | 4 +- test/test_js_dict_sender.js | 2 +- test/test_js_file_sender.js | 2 +- test/test_js_mix_payload_sender.js | 2 +- test/test_js_table_sender.js | 2 +- test/test_js_text_sender.js | 2 +- test/test_julia_dict_sender.jl | 2 +- test/test_julia_file_sender.jl | 2 +- test/test_julia_mix_payloads_sender.jl | 2 +- test/test_julia_table_sender.jl | 2 +- test/test_julia_text_sender.jl | 2 +- test/test_micropython_dict_sender.py | 2 +- test/test_micropython_file_sender.py | 2 +- test/test_micropython_mixed_sender.py | 2 +- test/test_micropython_text_sender.py | 2 +- 21 files changed, 65 insertions(+), 65 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index e46a9fc..dd3314a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -392,9 +392,9 @@ function smartsend( ``` **Return Value:** -- Returns a tuple `(env, msg_json_str)` where: +- Returns a tuple `(env, env_json_str)` where: - `env::msgEnvelope_v1` - The envelope object containing all metadata and payloads - - `msg_json_str::String` - JSON string representation of the envelope for publishing + - `env_json_str::String` - JSON string representation of the envelope for publishing **Options:** - `is_publish::Bool = true` - When `true` (default), the message is automatically published to NATS. When `false`, the function returns the envelope and JSON string without publishing, allowing manual publishing via NATS request-reply pattern. diff --git a/docs/implementation.md b/docs/implementation.md index b2e5d97..1ec9f2d 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -151,9 +151,9 @@ All three implementations (Julia, JavaScript, Python/Micropython) follow the sam The `smartsend` function now returns a tuple containing both the envelope object and the JSON string representation: ```julia -env, msg_json_str = smartsend(...) +env, env_json_str = smartsend(...) # env::msgEnvelope_v1 - The envelope object with all metadata and payloads -# msg_json_str::String - JSON string for publishing to NATS +# env_json_str::String - JSON string for publishing to NATS ``` **Options:** @@ -360,9 +360,9 @@ df = DataFrame( ) # Send via SmartSend - wrapped in a list (type is part of each tuple) -env, msg_json_str = SmartSend("analysis_results", [("table_data", df, "table")]) +env, env_json_str = SmartSend("analysis_results", [("table_data", df, "table")]) # env: msgEnvelope_v1 object with all metadata and payloads -# msg_json_str: JSON string representation of the envelope for publishing +# env_json_str: JSON string representation of the envelope for publishing ``` #### JavaScript (Receiver) diff --git a/examples/tutorial.md b/examples/tutorial.md index 8c874c4..9a4b89d 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -109,13 +109,13 @@ from nats_bridge import smartsend # Send a text message (is_publish=True by default) data = [("message", "Hello World", "text")] -env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") print("Message sent!") # Or use is_publish=False to get envelope and JSON without publishing -env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222", is_publish=False) +env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222", is_publish=False) # env: MessageEnvelope object -# msg_json_str: JSON string for publishing to NATS +# env_json_str: JSON string for publishing to NATS ``` #### JavaScript @@ -131,11 +131,11 @@ await smartsend("/chat/room1", [ console.log("Message sent!"); // Or use isPublish=false to get envelope and JSON without publishing -const { env, msg_json_str } = await smartsend("/chat/room1", [ +const { env, env_json_str } = await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } ], { natsUrl: "nats://localhost:4222", isPublish: false }); // env: MessageEnvelope object -// msg_json_str: JSON string for publishing to NATS +// env_json_str: JSON string for publishing to NATS ``` #### Julia @@ -145,9 +145,9 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") # env: msgEnvelope_v1 object with all metadata and payloads -# msg_json_str: JSON string representation of the envelope for publishing +# env_json_str: JSON string representation of the envelope for publishing println("Message sent!") ``` @@ -208,7 +208,7 @@ config = { # Send as dictionary type data = [("config", config, "dictionary")] -env, msg_json_str = smartsend("/device/config", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/device/config", data, nats_url="nats://localhost:4222") ``` #### JavaScript @@ -222,7 +222,7 @@ const config = { update_interval: 60 }; -const { env, msg_json_str } = await smartsend("/device/config", [ +const { env, env_json_str } = await smartsend("/device/config", [ { dataname: "config", data: config, type: "dictionary" } ]); ``` @@ -239,7 +239,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -env, msg_json_str = smartsend("/device/config", data) +env, env_json_str = smartsend("/device/config", data) ``` ### Example 2: Sending Binary Data (Image) @@ -255,7 +255,7 @@ with open("image.png", "rb") as f: # Send as binary type data = [("user_image", image_data, "binary")] -env, msg_json_str = smartsend("/chat/image", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/image", data, nats_url="nats://localhost:4222") ``` #### JavaScript @@ -267,7 +267,7 @@ const { smartsend } = require('./src/NATSBridge'); const fs = require('fs'); const image_data = fs.readFileSync('image.png'); -const { env, msg_json_str } = await smartsend("/chat/image", [ +const { env, env_json_str } = await smartsend("/chat/image", [ { dataname: "user_image", data: image_data, type: "binary" } ]); ``` @@ -281,7 +281,7 @@ using NATSBridge image_data = read("image.png") data = [("user_image", image_data, "binary")] -env, msg_json_str = smartsend("/chat/image", data) +env, env_json_str = smartsend("/chat/image", data) ``` ### Example 3: Request-Response Pattern @@ -293,7 +293,7 @@ from nats_bridge import smartsend # Send command with reply-to data = [("command", {"action": "read_sensor"}, "dictionary")] -env, msg_json_str = smartsend( +env, env_json_str = smartsend( "/device/command", data, nats_url="nats://localhost:4222", @@ -301,7 +301,7 @@ env, msg_json_str = smartsend( reply_to_msg_id="cmd-001" ) # env: msgEnvelope_v1 object -# msg_json_str: JSON string for publishing to NATS +# env_json_str: JSON string for publishing to NATS ``` #### JavaScript (Responder) @@ -358,7 +358,7 @@ import os large_data = os.urandom(2_000_000) # 2MB of random data # Send with file server URL -env, msg_json_str = smartsend( +env, env_json_str = smartsend( "/data/large", [("large_file", large_data, "binary")], nats_url="nats://localhost:4222", @@ -380,7 +380,7 @@ const largeData = new ArrayBuffer(2_000_000); const view = new Uint8Array(largeData); view.fill(42); // Fill with some data -const { env, msg_json_str } = await smartsend("/data/large", [ +const { env, env_json_str } = await smartsend("/data/large", [ { dataname: "large_file", data: largeData, type: "binary" } ], { fileserverUrl: "http://localhost:8080", @@ -396,7 +396,7 @@ using NATSBridge # Create large data (> 1MB) large_data = rand(UInt8, 2_000_000) -env, msg_json_str = smartsend( +env, env_json_str = smartsend( "/data/large", [("large_file", large_data, "binary")], fileserver_url="http://localhost:8080" @@ -425,7 +425,7 @@ data = [ ("user_avatar", image_data, "image") ] -env, msg_json_str = smartsend("/chat/mixed", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/mixed", data, nats_url="nats://localhost:4222") ``` #### JavaScript @@ -435,7 +435,7 @@ const { smartsend } = require('./src/NATSBridge'); const fs = require('fs'); -const { env, msg_json_str } = await smartsend("/chat/mixed", [ +const { env, env_json_str } = await smartsend("/chat/mixed", [ { dataname: "message_text", data: "Hello with image!", @@ -461,7 +461,7 @@ data = [ ("user_avatar", image_data, "image") ] -env, msg_json_str = smartsend("/chat/mixed", data) +env, env_json_str = smartsend("/chat/mixed", data) ``` ### Example 6: Table Data (Arrow IPC) @@ -483,7 +483,7 @@ df = pd.DataFrame({ # Send as table type data = [("students", df, "table")] -env, msg_json_str = smartsend("/data/students", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/data/students", data, nats_url="nats://localhost:4222") ``` #### Julia @@ -500,7 +500,7 @@ df = DataFrame( ) data = [("students", df, "table")] -env, msg_json_str = smartsend("/data/students", data) +env, env_json_str = smartsend("/data/students", data) ``` --- @@ -519,7 +519,7 @@ using NATSBridge # Send dictionary from Julia to JavaScript config = Dict("step_size" => 0.01, "iterations" => 1000) data = [("config", config, "dictionary")] -env, msg_json_str = smartsend("/analysis/config", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/analysis/config", data, nats_url="nats://localhost:4222") ``` #### JavaScript Receiver @@ -544,7 +544,7 @@ for (const payload of envelope.payloads) { ```javascript const { smartsend } = require('./src/NATSBridge'); -const { env, msg_json_str } = await smartsend("/data/transfer", [ +const { env, env_json_str } = await smartsend("/data/transfer", [ { dataname: "message", data: "Hello from JS!", type: "text" } ]); ``` @@ -568,7 +568,7 @@ for dataname, data, type in envelope["payloads"]: from nats_bridge import smartsend data = [("message", "Hello from Python!", "text")] -env, msg_json_str = smartsend("/chat/python", data) +env, env_json_str = smartsend("/chat/python", data) ``` #### Julia Receiver diff --git a/examples/walkthrough.md b/examples/walkthrough.md index 5bcb566..be72fd2 100644 --- a/examples/walkthrough.md +++ b/examples/walkthrough.md @@ -132,7 +132,7 @@ class ChatUI { }); } - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( `/chat/${this.currentRoom}`, data, { @@ -304,7 +304,7 @@ class FileUploadService { type: 'binary' }]; - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( `/files/${recipient}`, data, { @@ -534,7 +534,7 @@ class SensorSender: data = [("reading", reading.to_dict(), "dictionary")] # With is_publish=False, returns (envelope, json_str) without publishing - env, msg_json_str = smartsend( + env, env_json_str = smartsend( f"/sensors/{sensor_id}/prepare", data, nats_url=self.nats_url, @@ -543,9 +543,9 @@ class SensorSender: ) # Now you can publish manually using NATS request-reply pattern - # nc.request(subject, msg_json_str, reply_to=reply_to_topic) + # nc.request(subject, env_json_str, reply_to=reply_to_topic) - return env, msg_json_str + return env, env_json_str def send_batch(self, readings: List[SensorReading]): batch = SensorBatch() @@ -833,7 +833,7 @@ class DashboardUI { async refreshData() { // Request fresh data - const { env, msg_json_str } = await smartsend("/dashboard/request", [ + const { env, env_json_str } = await smartsend("/dashboard/request", [ { dataname: "request", data: { type: "refresh" }, type: "dictionary" } ], { fileserverUrl: window.config.fileserver_url diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index d5aae71..7d4549a 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -386,9 +386,9 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c - `is_publish::Bool = true` - Whether to automatically publish the message to NATS # Return: - - A tuple `(env, msg_json_str)` where: + - A tuple `(env, env_json_str)` where: - `env::msgEnvelope_v1` - The envelope object containing all metadata and payloads - - `msg_json_str::String` - JSON string representation of the envelope for publishing + - `env_json_str::String` - JSON string representation of the envelope for publishing # Example ```jldoctest @@ -415,7 +415,7 @@ env, msg_json = smartsend("chat.subject", [ ]) # Publish the JSON string directly using NATS request-reply pattern -# reply = NATS.request(nats_url, subject, msg_json_str; reply_to=reply_to_topic) +# reply = NATS.request(nats_url, subject, env_json_str; reply_to=reply_to_topic) ``` """ #[PENDING] function smartsend( @@ -432,7 +432,7 @@ function smartsend( receiver_id::String = "", reply_to::String = "", reply_to_msg_id::String = "", - is_publish::Bool = true # some time the user want to get env and msg_json_str from this function without publishing the msg + is_publish::Bool = true # some time the user want to get env and env_json_str from this function without publishing the msg ) where {T1<:Any} # Generate correlation ID if not provided @@ -516,12 +516,12 @@ function smartsend( metadata = Dict{String, Any}(), ) - msg_json_str = envelope_to_json(env) # Convert envelope to JSON + env_json_str = envelope_to_json(env) # Convert envelope to JSON if is_publish - publish_message(nats_url, subject, msg_json_str, cid) # Publish message to NATS + publish_message(nats_url, subject, env_json_str, cid) # Publish message to NATS end - return (env, msg_json_str) + return (env, env_json_str) end diff --git a/src/NATSBridge.js b/src/NATSBridge.js index c45f5e8..536e123 100644 --- a/src/NATSBridge.js +++ b/src/NATSBridge.js @@ -462,7 +462,7 @@ async function smartsend(subject, data, options = {}) { * @param {string} options.replyToMsgId - Message ID this message is replying to (default: "") * @param {boolean} options.isPublish - Whether to automatically publish the message to NATS (default: true) * - * @returns {Promise} - An object with { env: MessageEnvelope, msg_json_str: string } + * @returns {Promise} - An object with { env: MessageEnvelope, env_json_str: string } */ const { natsUrl = DEFAULT_NATS_URL, @@ -559,17 +559,17 @@ async function smartsend(subject, data, options = {}) { }); // Convert envelope to JSON string - const msg_json_str = env.toString(); + const env_json_str = env.toString(); // Publish to NATS if isPublish is true if (isPublish) { - await publish_message(natsUrl, subject, msg_json_str, correlationId); + await publish_message(natsUrl, subject, env_json_str, correlationId); } // Return both envelope and JSON string (tuple-like structure) return { env: env, - msg_json_str: msg_json_str + env_json_str: env_json_str }; } diff --git a/src/nats_bridge.py b/src/nats_bridge.py index 63ebedc..7e73473 100644 --- a/src/nats_bridge.py +++ b/src/nats_bridge.py @@ -462,9 +462,9 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F is_publish: Whether to automatically publish the message to NATS (default: True) Returns: - tuple: (env, msg_json_str) where: + tuple: (env, env_json_str) where: - env: MessageEnvelope object with all metadata and payloads - - msg_json_str: JSON string representation of the envelope for publishing + - env_json_str: JSON string representation of the envelope for publishing """ # Generate correlation ID if not provided cid = correlation_id if correlation_id else str(uuid.uuid4()) diff --git a/test/test_js_dict_sender.js b/test/test_js_dict_sender.js index 99b0912..6da3c21 100644 --- a/test/test_js_dict_sender.js +++ b/test/test_js_dict_sender.js @@ -118,7 +118,7 @@ async function test_dict_send() { // Use smartsend with dictionary type // For small Dictionary: will use direct transport (JSON encoded) // For large Dictionary: will use link transport (uploaded to fileserver) - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( SUBJECT, [data1, data2], { diff --git a/test/test_js_file_sender.js b/test/test_js_file_sender.js index 80b7742..383553c 100644 --- a/test/test_js_file_sender.js +++ b/test/test_js_file_sender.js @@ -98,7 +98,7 @@ async function test_large_binary_send() { // Use smartsend with binary type - will automatically use link transport // if file size exceeds the threshold (1MB by default) - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( SUBJECT, [data1, data2], { diff --git a/test/test_js_mix_payload_sender.js b/test/test_js_mix_payload_sender.js index e25eeb3..9c02b94 100644 --- a/test/test_js_mix_payload_sender.js +++ b/test/test_js_mix_payload_sender.js @@ -222,7 +222,7 @@ async function test_mix_send() { ]; // Use smartsend with mixed content - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( SUBJECT, payloads, { diff --git a/test/test_js_table_sender.js b/test/test_js_table_sender.js index 2aedabd..0ebb1b3 100644 --- a/test/test_js_table_sender.js +++ b/test/test_js_table_sender.js @@ -118,7 +118,7 @@ async function test_table_send() { // Use smartsend with table type // For small Table: will use direct transport (Arrow IPC encoded) // For large Table: will use link transport (uploaded to fileserver) - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( SUBJECT, [data1, data2], { diff --git a/test/test_js_text_sender.js b/test/test_js_text_sender.js index 81088cc..83e05cc 100644 --- a/test/test_js_text_sender.js +++ b/test/test_js_text_sender.js @@ -94,7 +94,7 @@ async function test_text_send() { // Use smartsend with text type // For small text: will use direct transport (Base64 encoded UTF-8) // For large text: will use link transport (uploaded to fileserver) - const { env, msg_json_str } = await smartsend( + const { env, env_json_str } = await smartsend( SUBJECT, [data1, data2], { diff --git a/test/test_julia_dict_sender.jl b/test/test_julia_dict_sender.jl index 75e6aff..c180320 100644 --- a/test/test_julia_dict_sender.jl +++ b/test/test_julia_dict_sender.jl @@ -92,7 +92,7 @@ function test_dict_send() # Use smartsend with dictionary type # For small Dictionary: will use direct transport (JSON encoded) # For large Dictionary: will use link transport (uploaded to fileserver) - env, msg_json = NATSBridge.smartsend( + env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL, diff --git a/test/test_julia_file_sender.jl b/test/test_julia_file_sender.jl index a4b086f..db1bed1 100644 --- a/test/test_julia_file_sender.jl +++ b/test/test_julia_file_sender.jl @@ -79,7 +79,7 @@ function test_large_binary_send() # Use smartsend with binary type - will automatically use link transport # if file size exceeds the threshold (1MB by default) # API: smartsend(subject, [(dataname, data, type), ...]; keywords...) - env, msg_json = NATSBridge.smartsend( + env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL; diff --git a/test/test_julia_mix_payloads_sender.jl b/test/test_julia_mix_payloads_sender.jl index c1805b7..f62e009 100644 --- a/test/test_julia_mix_payloads_sender.jl +++ b/test/test_julia_mix_payloads_sender.jl @@ -186,7 +186,7 @@ function test_mix_send() ] # Use smartsend with mixed content - env, msg_json = NATSBridge.smartsend( + env, env_json_str = NATSBridge.smartsend( SUBJECT, payloads; # List of (dataname, data, type) tuples nats_url = NATS_URL, diff --git a/test/test_julia_table_sender.jl b/test/test_julia_table_sender.jl index 338e73f..75e93f3 100644 --- a/test/test_julia_table_sender.jl +++ b/test/test_julia_table_sender.jl @@ -90,7 +90,7 @@ function test_table_send() # Use smartsend with table type # For small DataFrame: will use direct transport (Base64 encoded Arrow IPC) # For large DataFrame: will use link transport (uploaded to fileserver) - env, msg_json = NATSBridge.smartsend( + env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL, diff --git a/test/test_julia_text_sender.jl b/test/test_julia_text_sender.jl index 4746f82..6625d16 100644 --- a/test/test_julia_text_sender.jl +++ b/test/test_julia_text_sender.jl @@ -75,7 +75,7 @@ function test_text_send() # Use smartsend with text type # For small text: will use direct transport (Base64 encoded UTF-8) # For large text: will use link transport (uploaded to fileserver) - env, msg_json = NATSBridge.smartsend( + env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples nats_url = NATS_URL, diff --git a/test/test_micropython_dict_sender.py b/test/test_micropython_dict_sender.py index abdf197..bdc5878 100644 --- a/test/test_micropython_dict_sender.py +++ b/test/test_micropython_dict_sender.py @@ -64,7 +64,7 @@ def main(): log_trace(correlation_id, f"Correlation ID: {correlation_id}") # Use smartsend with dictionary type - env, msg_json = smartsend( + env, env_json_str = smartsend( SUBJECT, [data1, data2], # List of (dataname, data, type) tuples nats_url=NATS_URL, diff --git a/test/test_micropython_file_sender.py b/test/test_micropython_file_sender.py index dc06014..55b53bb 100644 --- a/test/test_micropython_file_sender.py +++ b/test/test_micropython_file_sender.py @@ -44,7 +44,7 @@ def main(): log_trace(correlation_id, f"Correlation ID: {correlation_id}") # Use smartsend with binary type - env, msg_json = smartsend( + env, env_json_str = smartsend( SUBJECT, [data1, data2], # List of (dataname, data, type) tuples nats_url=NATS_URL, diff --git a/test/test_micropython_mixed_sender.py b/test/test_micropython_mixed_sender.py index 437087b..66a57a3 100644 --- a/test/test_micropython_mixed_sender.py +++ b/test/test_micropython_mixed_sender.py @@ -58,7 +58,7 @@ def main(): log_trace(correlation_id, f"Correlation ID: {correlation_id}") # Use smartsend with mixed types - env, msg_json = smartsend( + env, env_json_str = smartsend( SUBJECT, data, # List of (dataname, data, type) tuples nats_url=NATS_URL, diff --git a/test/test_micropython_text_sender.py b/test/test_micropython_text_sender.py index c048216..6aaf7f9 100644 --- a/test/test_micropython_text_sender.py +++ b/test/test_micropython_text_sender.py @@ -46,7 +46,7 @@ def main(): # Use smartsend with text type # For small text: will use direct transport (Base64 encoded UTF-8) # For large text: will use link transport (uploaded to fileserver) - env, msg_json = smartsend( + env, env_json_str = smartsend( SUBJECT, [data1, data2], # List of (dataname, data, type) tuples nats_url=NATS_URL, -- 2.49.1 From 2c340e37c7e5d567993221aced68e772a05f9901 Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 22:00:06 +0700 Subject: [PATCH 09/35] update --- README.md | 90 +++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index c48cc8a..6b81012 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ from nats_bridge import smartsend # Send a text message data = [("message", "Hello World", "text")] -envelope, msg_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") print("Message sent!") ``` @@ -183,7 +183,7 @@ print("Message sent!") const { smartsend } = require('./src/NATSBridge'); // Send a text message -const { envelope, msg_json_str } = await smartsend("/chat/room1", [ +const { env, env_json_str } = await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } ], { natsUrl: "nats://localhost:4222" }); @@ -197,7 +197,7 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -envelope, msg_json_str = NATSBridge.smartsend("/chat/room1", data; nats_url="nats://localhost:4222") +env, env_json_str = NATSBridge.smartsend("/chat/room1", data; nats_url="nats://localhost:4222") println("Message sent!") ``` @@ -221,8 +221,8 @@ async def main(): # Subscribe to the subject - msg comes from the callback async def message_handler(msg): # Receive and process message - envelope = smartreceive(msg.data) - for dataname, data, type in envelope["payloads"]: + env = smartreceive(msg.data) + for dataname, data, type in env["payloads"]: print(f"Received {dataname}: {data}") sid = await nc.subscribe(SUBJECT, cb=message_handler) @@ -251,8 +251,8 @@ async function main() { for await (const msg of sub) { // Receive and process message - const envelope = await smartreceive(msg); - for (const payload of envelope.payloads) { + const env = await smartreceive(msg); + for (const payload of env.payloads) { console.log(`Received ${payload.dataname}: ${payload.data}`); } } @@ -283,8 +283,8 @@ function test_receive() log_trace("Received message on $(msg.subject)") # Receive and process message - envelope, msg_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) - for (dataname, data, type) in envelope["payloads"] + env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + for (dataname, data, type) in env["payloads"] println("Received $dataname: $data") end end @@ -310,7 +310,7 @@ Sends data either directly via NATS or via a fileserver URL, depending on payloa ```python from nats_bridge import smartsend -envelope, msg_json_str = smartsend( +env, env_json_str = smartsend( subject, # NATS subject to publish to data, # List of (dataname, data, type) tuples nats_url="nats://localhost:4222", # NATS server URL @@ -333,7 +333,7 @@ envelope, msg_json_str = smartsend( ```javascript const { smartsend } = require('./src/NATSBridge'); -const { envelope, msg_json_str } = await smartsend( +const { env, env_json_str } = await smartsend( subject, // NATS subject data, // Array of {dataname, data, type} { @@ -358,7 +358,7 @@ const { envelope, msg_json_str } = await smartsend( ```julia using NATSBridge -envelope, msg_json_str = NATSBridge.smartsend( +env, env_json_str = NATSBridge.smartsend( subject, # NATS subject data::AbstractArray{Tuple{String, Any, String}}; # List of (dataname, data, type) nats_url::String = "nats://localhost:4222", @@ -375,8 +375,8 @@ envelope, msg_json_str = NATSBridge.smartsend( is_publish::Bool = true # Whether to automatically publish to NATS ) # Returns: (msgEnvelope_v1, JSON string) -# - envelope: msgEnvelope_v1 object with all envelope metadata and payloads -# - msg_json_str: JSON string representation of the envelope for publishing +# - env: msgEnvelope_v1 object with all envelope metadata and payloads +# - env_json_str: JSON string representation of the envelope for publishing ``` ### smartreceive @@ -389,7 +389,7 @@ Receives and processes messages from NATS, handling both direct and link transpo from nats_bridge import smartreceive # Note: For nats-py, use msg.data to pass the raw message data -envelope = smartreceive( +env = smartreceive( msg.data, # NATS message data (msg.data for nats-py) fileserver_download_handler=_fetch_with_backoff, # Download handler max_retries=5, # Max retry attempts @@ -405,7 +405,7 @@ envelope = smartreceive( const { smartreceive } = require('./src/NATSBridge'); // Note: msg is the NATS message object from subscription -const envelope = await smartreceive( +const env = await smartreceive( msg, // NATS message (raw object from subscription) { fileserverDownloadHandler: customDownloadHandler, @@ -423,7 +423,7 @@ const envelope = await smartreceive( using NATSBridge # Note: msg is a NATS.Msg object passed from the subscription callback -envelope, msg_json_str = NATSBridge.smartreceive( +env, env_json_str = NATSBridge.smartreceive( msg::NATS.Msg; fileserverDownloadHandler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -517,14 +517,14 @@ data = [ ("large_document", large_file_data, "binary") ] -envelope, msg_json_str = smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") +env, env_json_str = smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") ``` #### JavaScript ```javascript const { smartsend } = require('./src/NATSBridge'); -const { envelope, msg_json_str } = await smartsend("/chat/room1", [ +const { env, env_json_str } = await smartsend("/chat/room1", [ { dataname: "message_text", data: "Hello!", type: "text" }, { dataname: "user_avatar", data: image_data, type: "image" }, { dataname: "large_document", data: large_file_data, type: "binary" } @@ -543,7 +543,7 @@ data = [ ("large_document", large_file_data, "binary") ] -envelope, msg_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") +env, env_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") ``` ### Example 2: Dictionary Exchange @@ -561,7 +561,7 @@ config = { } data = [("config", config, "dictionary")] -envelope, msg_json_str = smartsend("/device/config", data) +env, env_json_str = smartsend("/device/config", data) ``` #### JavaScript @@ -574,7 +574,7 @@ const config = { update_interval: 60 }; -const { envelope, msg_json_str } = await smartsend("/device/config", [ +const { env, env_json_str } = await smartsend("/device/config", [ { dataname: "config", data: config, type: "dictionary" } ]); ``` @@ -590,7 +590,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -envelope, msg_json_str = NATSBridge.smartsend("/device/config", data) +env, env_json_str = NATSBridge.smartsend("/device/config", data) ``` ### Example 3: Table Data (Arrow IPC) @@ -609,7 +609,7 @@ df = pd.DataFrame({ }) data = [("students", df, "table")] -envelope, msg_json_str = smartsend("/data/analysis", data) +env, env_json_str = smartsend("/data/analysis", data) ``` #### JavaScript @@ -622,7 +622,7 @@ const tableData = [ { id: 3, name: "Charlie", score: 92 } ]; -const { envelope, msg_json_str } = await smartsend("/data/analysis", [ +const { env, env_json_str } = await smartsend("/data/analysis", [ { dataname: "students", data: tableData, type: "table" } ]); ``` @@ -639,7 +639,7 @@ df = DataFrame( ) data = [("students", df, "table")] -envelope, msg_json_str = NATSBridge.smartsend("/data/analysis", data) +env, env_json_str = NATSBridge.smartsend("/data/analysis", data) ``` ### Example 4: Request-Response Pattern with Envelope JSON @@ -650,16 +650,16 @@ Bi-directional communication with reply-to support. The `smartsend` function now ```python from nats_bridge import smartsend -envelope, msg_json_str = smartsend( +env, env_json_str = smartsend( "/device/command", [("command", {"action": "read_sensor"}, "dictionary")], reply_to="/device/response" ) -# envelope: msgEnvelope_v1 object -# msg_json_str: JSON string for publishing to NATS +# env: msgEnvelope_v1 object +# env_json_str: JSON string for publishing to NATS -# The msg_json_str can also be published directly using NATS request-reply pattern -# nc.request("/device/command", msg_json_str, reply_to="/device/response") +# The env_json_str can also be published directly using NATS request-reply pattern +# nc.request("/device/command", env_json_str, reply_to="/device/response") ``` #### Python/Micropython (Responder) @@ -677,8 +677,8 @@ async def main(): nc = await nats.connect(NATS_URL) async def message_handler(msg): - envelope = smartreceive(msg.data) - for dataname, data, type in envelope["payloads"]: + env = smartreceive(msg.data) + for dataname, data, type in env["payloads"]: if data.get("action") == "read_sensor": response = {"sensor_id": "sensor-001", "value": 42.5} smartsend(REPLY_SUBJECT, [("data", response, "dictionary")]) @@ -694,7 +694,7 @@ asyncio.run(main()) ```javascript const { smartsend } = require('./src/NATSBridge'); -const { envelope, msg_json_str } = await smartsend("/device/command", [ +const { env, env_json_str } = await smartsend("/device/command", [ { dataname: "command", data: { action: "read_sensor" }, type: "dictionary" } ], { replyTo: "/device/response" @@ -717,8 +717,8 @@ async function main() { const sub = nc.subscribe(SUBJECT); for await (const msg of sub) { - const envelope = await smartreceive(msg); - for (const payload of envelope.payloads) { + const env = await smartreceive(msg); + for (const payload of env.payloads) { if (payload.dataname === "command" && payload.data.action === "read_sensor") { const response = { sensor_id: "sensor-001", value: 42.5 }; await smartsend(REPLY_SUBJECT, [ @@ -736,7 +736,7 @@ main(); ```julia using NATSBridge -envelope, msg_json_str = NATSBridge.smartsend( +env, env_json_str = NATSBridge.smartsend( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; reply_to="/device/response" @@ -755,8 +755,8 @@ const NATS_URL = "nats://localhost:4222" function test_responder() conn = NATS.connect(NATS_URL) NATS.subscribe(conn, SUBJECT) do msg - envelope, msg_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) - for (dataname, data, type) in envelope["payloads"] + env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + for (dataname, data, type) in env["payloads"] if dataname == "command" && data["action"] == "read_sensor" response = Dict("sensor_id" => "sensor-001", "value" => 42.5) smartsend(REPLY_SUBJECT, [("data", response, "dictionary")]) @@ -794,8 +794,8 @@ async def main(): # Receive commands - msg comes from the callback async def message_handler(msg): - envelope = smartreceive(msg.data) - for dataname, data, type in envelope["payloads"]: + env = smartreceive(msg.data) + for dataname, data, type in env["payloads"]: if type == "dictionary" and data.get("action") == "reboot": # Execute reboot pass @@ -822,8 +822,8 @@ async function main() { const sub = nc.subscribe(SUBJECT); for await (const msg of sub) { - const envelope = await smartreceive(msg); - for (const payload of envelope.payloads) { + const env = await smartreceive(msg); + for (const payload of env.payloads) { if (payload.dataname === "temperature") { console.log(`Temperature: ${payload.data}`); } else if (payload.dataname === "humidity") { @@ -847,8 +847,8 @@ const NATS_URL = "nats://localhost:4222" function test_receiver() conn = NATS.connect(NATS_URL) NATS.subscribe(conn, SUBJECT) do msg - envelope, msg_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) - for (dataname, data, type) in envelope["payloads"] + env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + for (dataname, data, type) in env["payloads"] if dataname == "temperature" println("Temperature: $data") elseif dataname == "humidity" -- 2.49.1 From 64c62e616bab1950c77476255d4efbdc60ddc5ee Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 23 Feb 2026 22:06:57 +0700 Subject: [PATCH 10/35] update --- docs/architecture.md | 6 ++--- docs/implementation.md | 50 ++++++++++++++++++++--------------------- examples/tutorial.md | 32 +++++++++++++------------- examples/walkthrough.md | 32 +++++++++++++------------- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index dd3314a..fee3cb2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -103,9 +103,9 @@ smartsend( ) # Receive returns a dictionary envelope with all metadata and deserialized payloads -envelope = smartreceive(msg, fileserverDownloadHandler, max_retries, base_delay, max_delay) -# envelope["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...] -# envelope["correlationId"], envelope["msgId"], etc. +env = smartreceive(msg, fileserverDownloadHandler, max_retries, base_delay, max_delay) +# env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...] +# env["correlationId"], env["msgId"], etc. ``` ## Architecture Diagram diff --git a/docs/implementation.md b/docs/implementation.md index 1ec9f2d..9a02d5d 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -59,9 +59,9 @@ smartsend("/test", [(dataname1, data1, "text")], ...) smartsend("/test", [(dataname1, data1, "dictionary"), (dataname2, data2, "table")], ...) # Receive returns a dictionary envelope with all metadata and deserialized payloads -envelope = smartreceive(msg, ...) -# envelope["payloads"] = [(dataname1, data1, "text"), (dataname2, data2, "table"), ...] -# envelope["correlationId"], envelope["msgId"], etc. +env = smartreceive(msg, ...) +# env["payloads"] = [(dataname1, data1, "text"), (dataname2, data2, "table"), ...] +# env["correlationId"], env["msgId"], etc. ``` ## Cross-Platform Interoperability @@ -104,8 +104,8 @@ smartsend("/cross_platform", data, nats_url="nats://localhost:4222") ```javascript // JavaScript receiver const { smartreceive } = require('./src/NATSBridge'); -const envelope = await smartreceive(msg); -// envelope.payloads[0].data === "Hello from Julia!" +const env = await smartreceive(msg); +// env.payloads[0].data === "Hello from Julia!" ``` ```python @@ -330,18 +330,18 @@ const nc = await connect({ servers: ['nats://localhost:4222'] }); const sub = nc.subscribe("control"); for await (const msg of sub) { - const envelope = await smartreceive(msg); + const env = await smartreceive(msg); // Process the payloads from the envelope - for (const payload of envelope.payloads) { + for (const payload of env.payloads) { const { dataname, data, type } = payload; console.log(`Received ${dataname} of type ${type}`); console.log(`Data: ${JSON.stringify(data)}`); } // Also access envelope metadata - console.log(`Correlation ID: ${envelope.correlationId}`); - console.log(`Message ID: ${envelope.msgId}`); + console.log(`Correlation ID: ${env.correlationId}`); + console.log(`Message ID: ${env.msgId}`); } ``` @@ -369,11 +369,11 @@ env, env_json_str = SmartSend("analysis_results", [("table_data", df, "table")]) ```javascript const { smartreceive } = require('./src/NATSBridge'); -const envelope = await smartreceive(msg); +const env = await smartreceive(msg); // Use table data from the payloads field // Note: Tables are sent as arrays of objects in JavaScript -const table = envelope.payloads; +const table = env.payloads; ``` ### Scenario 3: Live Binary Processing @@ -423,10 +423,10 @@ from nats_bridge import smartreceive # Receive binary data def process_binary(msg): - envelope = smartreceive(msg) + env = smartreceive(msg) - # Process the binary data from envelope.payloads - for dataname, data, type in envelope["payloads"]: + # Process the binary data from env.payloads + for dataname, data, type in env["payloads"]: if type == "binary": # data is bytes print(f"Received binary data: {dataname}, size: {len(data)}") @@ -439,10 +439,10 @@ const { smartreceive } = require('./src/NATSBridge'); // Receive binary data function process_binary(msg) { - const envelope = await smartreceive(msg); + const env = await smartreceive(msg); - // Process the binary data from envelope.payloads - for (const payload of envelope.payloads) { + // Process the binary data from env.payloads + for (const payload of env.payloads) { if (payload.type === "binary") { // data is an ArrayBuffer or Uint8Array console.log(`Received binary data: ${payload.dataname}, size: ${payload.data.length}`); @@ -483,8 +483,8 @@ const consumer = await js.pullSubscribe("health", { // Process historical and real-time messages for await (const msg of consumer) { - const envelope = await smartreceive(msg); - // envelope.payloads contains the list of payloads + const env = await smartreceive(msg); + // env.payloads contains the list of payloads // Each payload has: dataname, data, type msg.ack(); } @@ -501,10 +501,10 @@ import json # Device configuration handler def handle_device_config(msg): - envelope = smartreceive(msg) + env = smartreceive(msg) # Process configuration from payloads - for dataname, data, type in envelope["payloads"]: + for dataname, data, type in env["payloads"]: if type == "dictionary": print(f"Received configuration: {data}") # Apply configuration to device @@ -523,7 +523,7 @@ def handle_device_config(msg): "device/response", [("config", config, "dictionary")], nats_url="nats://localhost:4222", - reply_to=envelope.get("replyTo") + reply_to=env.get("replyTo") ) ``` @@ -583,11 +583,11 @@ smartsend( const { smartreceive, smartsend } = require('./src/NATSBridge'); // Receive NATS message with direct transport -const envelope = await smartreceive(msg); +const env = await smartreceive(msg); // Decode Base64 payload (for direct transport) -// For tables, data is in envelope.payloads -const table = envelope.payloads; // Array of objects +// For tables, data is in env.payloads +const table = env.payloads; // Array of objects // User makes selection const selection = uiComponent.getSelectedOption(); diff --git a/examples/tutorial.md b/examples/tutorial.md index 9a4b89d..220df84 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -159,8 +159,8 @@ println("Message sent!") from nats_bridge import smartreceive # Receive and process message -envelope = smartreceive(msg) -for dataname, data, type in envelope["payloads"]: +env = smartreceive(msg) +for dataname, data, type in env["payloads"]: print(f"Received {dataname}: {data}") ``` @@ -170,8 +170,8 @@ for dataname, data, type in envelope["payloads"]: const { smartreceive } = require('./src/NATSBridge'); // Receive and process message -const envelope = await smartreceive(msg); -for (const payload of envelope.payloads) { +const env = await smartreceive(msg); +for (const payload of env.payloads) { console.log(`Received ${payload.dataname}: ${payload.data}`); } ``` @@ -182,8 +182,8 @@ for (const payload of envelope.payloads) { using NATSBridge # Receive and process message -envelope = smartreceive(msg, fileserverDownloadHandler) -for (dataname, data, type) in envelope["payloads"] +env = smartreceive(msg, fileserverDownloadHandler) +for (dataname, data, type) in env["payloads"] println("Received $dataname: $data") end ``` @@ -313,10 +313,10 @@ const { smartreceive, smartsend } = require('./src/NATSBridge'); const sub = nc.subscribe("/device/command"); for await (const msg of sub) { - const envelope = await smartreceive(msg); + const env = await smartreceive(msg); // Process command - for (const payload of envelope.payloads) { + for (const payload of env.payloads) { if (payload.dataname === "command") { const command = payload.data; @@ -331,8 +331,8 @@ for await (const msg of sub) { await smartsend("/device/response", [ { dataname: "sensor_data", data: response, type: "dictionary" } ], { - reply_to: envelope.replyTo, - reply_to_msg_id: envelope.msgId + reply_to: env.replyTo, + reply_to_msg_id: env.msgId }); } } @@ -528,8 +528,8 @@ env, env_json_str = smartsend("/analysis/config", data, nats_url="nats://localho const { smartreceive } = require('./src/NATSBridge'); // Receive dictionary from Julia -const envelope = await smartreceive(msg); -for (const payload of envelope.payloads) { +const env = await smartreceive(msg); +for (const payload of env.payloads) { if (payload.type === "dictionary") { console.log("Received config:", payload.data); // payload.data = { step_size: 0.01, iterations: 1000 } @@ -554,8 +554,8 @@ const { env, env_json_str } = await smartsend("/data/transfer", [ ```python from nats_bridge import smartreceive -envelope = smartreceive(msg) -for dataname, data, type in envelope["payloads"]: +env = smartreceive(msg) +for dataname, data, type in env["payloads"]: if type == "text": print(f"Received from JS: {data}") ``` @@ -576,8 +576,8 @@ env, env_json_str = smartsend("/chat/python", data) ```julia using NATSBridge -envelope = smartreceive(msg, fileserverDownloadHandler) -for (dataname, data, type) in envelope["payloads"] +env = smartreceive(msg, fileserverDownloadHandler) +for (dataname, data, type) in env["payloads"] if type == "text" println("Received from Python: $data") end diff --git a/examples/walkthrough.md b/examples/walkthrough.md index be72fd2..b9f68fe 100644 --- a/examples/walkthrough.md +++ b/examples/walkthrough.md @@ -216,15 +216,15 @@ class ChatHandler { } async handleMessage(msg) { - const envelope = await smartreceive(msg, { + const env = await smartreceive(msg, { fileserverDownloadHandler: this.downloadFile.bind(this) }); // Extract sender info from envelope - const sender = envelope.senderName || 'Anonymous'; + const sender = env.senderName || 'Anonymous'; // Process each payload - for (const payload of envelope.payloads) { + for (const payload of env.payloads) { if (payload.type === 'text') { this.ui.addMessage(sender, payload.data); } else if (payload.type === 'image') { @@ -356,12 +356,12 @@ class FileDownloadService { async downloadFile(sender, downloadId) { // Subscribe to sender's file channel - const envelope = await smartreceive(msg, { + const env = await smartreceive(msg, { fileserverDownloadHandler: this.fetchFromUrl.bind(this) }); // Process each payload - for (const payload of envelope.payloads) { + for (const payload of env.payloads) { if (payload.type === 'binary') { const filePath = `/downloads/${payload.dataname}`; fs.writeFileSync(filePath, payload.data); @@ -422,9 +422,9 @@ async function uploadFile(config) { const fileService = new FileUploadService(config.nats_url, config.fileserver_url); try { - const envelope = await fileService.uploadFile(filePath, recipient); + const env = await fileService.uploadFile(filePath, recipient); console.log('Upload successful!'); - console.log(`File ID: ${envelope.payloads[0].id}`); + console.log(`File ID: ${env.payloads[0].id}`); } catch (error) { console.error('Upload failed:', error.message); } @@ -533,7 +533,7 @@ class SensorSender: data = [("reading", reading.to_dict(), "dictionary")] - # With is_publish=False, returns (envelope, json_str) without publishing + # With is_publish=False, returns (env, env_json_str) without publishing env, env_json_str = smartsend( f"/sensors/{sensor_id}/prepare", data, @@ -597,9 +597,9 @@ class SensorReceiver: self.fileserver_download_handler = fileserver_download_handler def process_reading(self, msg): - envelope = smartreceive(msg, self.fileserver_download_handler) + env = smartreceive(msg, self.fileserver_download_handler) - for dataname, data, data_type in envelope["payloads"]: + for dataname, data, data_type in env["payloads"]: if data_type == "dictionary": reading = SensorReading( sensor_id=data["sensor_id"], @@ -699,10 +699,10 @@ class DeviceBridge: # Poll for messages msg = self._poll_for_message() if msg: - envelope = smartreceive(msg) + env = smartreceive(msg) # Process payloads - for dataname, data, data_type in envelope["payloads"]: + for dataname, data, data_type in env["payloads"]: if dataname == "command": callback(data) @@ -798,9 +798,9 @@ class DashboardServer: def receive_selection(self, callback): def handler(msg): - envelope = smartreceive(msg) + env = smartreceive(msg) - for dataname, data, data_type in envelope["payloads"]: + for dataname, data, data_type in env["payloads"]: if data_type == "dictionary": callback(data) @@ -842,12 +842,12 @@ class DashboardUI { async fetchData() { // Subscribe to data updates - const envelope = await smartreceive(msg, { + const env = await smartreceive(msg, { fileserverDownloadHandler: this.fetchFromUrl.bind(this) }); // Process table data - for (const payload of envelope.payloads) { + for (const payload of env.payloads) { if (payload.type === 'table') { // Deserialize Arrow IPC this.data = this.deserializeArrow(payload.data); -- 2.49.1 From 90d81617ef83494de857bdbd32c8fa02c073f122 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 17:58:59 +0700 Subject: [PATCH 11/35] update --- src/NATSBridge.jl | 459 +++++++++++++++++++++++----------------------- 1 file changed, 228 insertions(+), 231 deletions(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 7d4549a..4677c53 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -12,10 +12,10 @@ # # ```jldoctest # # Upload handler - uploads data to file server and returns URL -# fileserverUploadHandler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} -# +# fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} +# # # Download handler - fetches data from file server URL with exponential backoff -# fileserverDownloadHandler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} +# fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} # ``` # # Multi-Payload Support (Standard API): @@ -35,24 +35,23 @@ module NATSBridge -using Revise using NATS, JSON, Arrow, HTTP, UUIDs, Dates, Base64, PrettyPrinting # ---------------------------------------------- 100 --------------------------------------------- # # Constants const DEFAULT_SIZE_THRESHOLD = 1_000_000 # 1MB - threshold for switching from direct to link transport -const DEFAULT_NATS_URL = "nats://localhost:4222" # Default NATS server URL +const DEFAULT_BROKER_URL = "nats://localhost:4222" # Default NATS server URL const DEFAULT_FILESERVER_URL = "http://localhost:8080" # Default HTTP file server URL for link transport -""" msgPayload_v1 - Internal message payload structure +""" msg_payload_v1 - Internal message payload structure This structure represents a single payload within a NATS message envelope. It supports both direct transport (base64-encoded data) and link transport (URL-based). # Arguments: - `id::String` - Unique identifier for this payload (e.g., "uuid4") - `dataname::String` - Name of the payload (e.g., "login_image") - - `type::String` - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" + - `payload_type::String` - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" - `transport::String` - Transport method: "direct" or "link" - `encoding::String` - Encoding method: "none", "json", "base64", "arrow-ipc" - `size::Integer` - Size of the payload in bytes (e.g., 15433) @@ -68,14 +67,14 @@ It supports both direct transport (base64-encoded data) and link transport (URL- - `metadata::Dict{String, T} = Dict{String, Any}()` - Metadata dictionary # Return: - - A msgPayload_v1 struct instance + - A msg_payload_v1 struct instance # Example ```jldoctest using UUIDs # Create a direct transport payload -payload = msgPayload_v1( +payload = msg_payload_v1( "Hello World", "text"; id = string(uuid4()), @@ -87,7 +86,7 @@ payload = msgPayload_v1( ) # Create a link transport payload -payload = msgPayload_v1( +payload = msg_payload_v1( "http://example.com/file.zip", "binary"; id = string(uuid4()), @@ -98,10 +97,10 @@ payload = msgPayload_v1( ) ``` """ -struct msgPayload_v1 +struct msg_payload_v1 id::String # id of this payload e.g. "uuid4" dataname::String # name of this payload e.g. "login_image" - type::String # this payload type. Can be "text | dictionary | table | image | audio | video | binary" + payload_type::String # this payload type. Can be "text | dictionary | table | image | audio | video | binary" transport::String # "direct | link" encoding::String # "none | json | base64 | arrow-ipc" size::Integer # data size in bytes e.g. 15433 @@ -110,9 +109,9 @@ struct msgPayload_v1 end # constructor -function msgPayload_v1( +function msg_payload_v1( data::Any, - type::String; + payload_type::String; id::String = "", dataname::String = string(uuid4()), transport::String = "direct", @@ -120,10 +119,10 @@ function msgPayload_v1( size::Integer = 0, metadata::Dict{String, T} = Dict{String, Any}() ) where {T<:Any} - return msgPayload_v1( + return msg_payload_v1( id, dataname, - type, + payload_type, transport, encoding, size, @@ -133,101 +132,101 @@ function msgPayload_v1( end -""" msgEnvelope_v1 - Internal message envelope structure +""" msg_envelope_v1 - Internal message envelope structure This structure represents a complete NATS message envelope containing multiple payloads with metadata for routing, tracing, and message context. # Arguments: - - `sendTo::String` - NATS subject/topic to publish the message to (e.g., "/agent/wine/api/v1/prompt") - - `payloads::AbstractArray{msgPayload_v1}` - List of payloads to include in the message + - `send_to::String` - NATS subject/topic to publish the message to (e.g., "/agent/wine/api/v1/prompt") + - `payloads::AbstractArray{msg_payload_v1}` - List of payloads to include in the message # Keyword Arguments: - - `correlationId::String = ""` - Unique identifier to track messages across systems; auto-generated if empty - - `msgId::String = ""` - Unique message identifier; auto-generated if empty + - `correlation_id::String = ""` - Unique identifier to track messages across systems; auto-generated if empty + - `msg_id::String = ""` - Unique message identifier; auto-generated if empty - `timestamp::String = string(Dates.now())` - Message publication timestamp - - `msgPurpose::String = ""` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. - - `senderName::String = ""` - Name of the sender (e.g., "agent-wine-web-frontend") - - `senderId::String = ""` - UUID of the sender; auto-generated if empty - - `receiverName::String = ""` - Name of the receiver (empty string means broadcast) - - `receiverId::String = ""` - UUID of the receiver (empty string means broadcast) - - `replyTo::String = ""` - Topic where receiver should reply (empty string if no reply expected) - - `replyToMsgId::String = ""` - Message ID this message is replying to - - `brokerURL::String = DEFAULT_NATS_URL` - NATS broker URL + - `msg_purpose::String = ""` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. + - `sender_name::String = ""` - Name of the sender (e.g., "agent-wine-web-frontend") + - `sender_id::String = ""` - UUID of the sender; auto-generated if empty + - `receiver_name::String = ""` - Name of the receiver (empty string means broadcast) + - `receiver_id::String = ""` - UUID of the receiver (empty string means broadcast) + - `reply_to::String = ""` - Topic where receiver should reply (empty string if no reply expected) + - `reply_to_msg_id::String = ""` - Message ID this message is replying to + - `broker_url::String = DEFAULT_BROKER_URL` - NATS broker URL - `metadata::Dict{String, Any} = Dict{String, Any}()` - Optional message-level metadata # Return: - - A msgEnvelope_v1 struct instance + - A msg_envelope_v1 struct instance # Example ```jldoctest using UUIDs, NATSBridge # Create payloads for the message -payload1 = msgPayload_v1("Hello", "text"; dataname="message", transport="direct", encoding="base64") -payload2 = msgPayload_v1("http://example.com/file.zip", "binary"; dataname="file", transport="link") +payload1 = msg_payload_v1("Hello", "text"; dataname="message", transport="direct", encoding="base64") +payload2 = msg_payload_v1("http://example.com/file.zip", "binary"; dataname="file", transport="link") # Create message envelope -env = msgEnvelope_v1( +env = msg_envelope_v1( "my.subject", [payload1, payload2]; - correlationId = string(uuid4()), - msgPurpose = "chat", - senderName = "my-app", - receiverName = "receiver-app", - replyTo = "reply.subject" + correlation_id = string(uuid4()), + msg_purpose = "chat", + sender_name = "my-app", + receiver_name = "receiver-app", + reply_to = "reply.subject" ) ``` """ -struct msgEnvelope_v1 - correlationId::String # Unique identifier to track messages across systems. Many senders can talk about the same topic. - msgId::String # this message id +struct msg_envelope_v1 + correlation_id::String # Unique identifier to track messages across systems. Many senders can talk about the same topic. + msg_id::String # this message id timestamp::String # message published timestamp. string(Dates.now()) - sendTo::String # topic/subject the sender sends to e.g. "/agent/wine/api/v1/prompt" - msgPurpose::String # purpose of this message e.g. "ACK | NACK | updateStatus | shutdown | ..." - senderName::String # sender name (String) e.g. "agent-wine-web-frontend" - senderId::String # sender id e.g. uuid4snakecase() - receiverName::String # msg receiver name (String) e.g. "agent-backend" - receiverId::String # msg receiver id, nothing means everyone in the topic e.g. uuid4snakecase() + send_to::String # topic/subject the sender sends to e.g. "/agent/wine/api/v1/prompt" + msg_purpose::String # purpose of this message e.g. "ACK | NACK | updateStatus | shutdown | ..." + sender_name::String # sender name (String) e.g. "agent-wine-web-frontend" + sender_id::String # sender id e.g. uuid4snakecase() + receiver_name::String # msg receiver name (String) e.g. "agent-backend" + receiver_id::String # msg receiver id, nothing means everyone in the topic e.g. uuid4snakecase() - replyTo::String # sender ask receiver to reply to this topic - replyToMsgId::String # the message id this message is replying to - brokerURL::String # mqtt/NATS server address + reply_to::String # sender ask receiver to reply to this topic + reply_to_msg_id::String # the message id this message is replying to + broker_url::String # mqtt/NATS server address metadata::Dict{String, Any} - payloads::AbstractArray{msgPayload_v1} # multiple payload store here + payloads::AbstractArray{msg_payload_v1} # multiple payload store here end # constructor -function msgEnvelope_v1( - sendTo::String, - payloads::AbstractArray{msgPayload_v1}; - correlationId::String = "", - msgId::String = "", +function msg_envelope_v1( + send_to::String, + payloads::AbstractArray{msg_payload_v1}; + correlation_id::String = "", + msg_id::String = "", timestamp::String = string(Dates.now()), - msgPurpose::String = "", - senderName::String = "", - senderId::String = "", - receiverName::String = "", - receiverId::String = "", - replyTo::String = "", - replyToMsgId::String = "", - brokerURL::String = DEFAULT_NATS_URL, + msg_purpose::String = "", + sender_name::String = "", + sender_id::String = "", + receiver_name::String = "", + receiver_id::String = "", + reply_to::String = "", + reply_to_msg_id::String = "", + broker_url::String = DEFAULT_BROKER_URL, metadata::Dict{String, Any} = Dict{String, Any}() ) - return msgEnvelope_v1( - correlationId, - msgId, + return msg_envelope_v1( + correlation_id, + msg_id, timestamp, - sendTo, - msgPurpose, - senderName, - senderId, - receiverName, - receiverId, - replyTo, - replyToMsgId, - brokerURL, + send_to, + msg_purpose, + sender_name, + sender_id, + receiver_name, + receiver_id, + reply_to, + reply_to_msg_id, + broker_url, metadata, payloads ) @@ -235,19 +234,19 @@ end -""" envelope_to_json - Convert msgEnvelope_v1 to JSON string -This function converts the msgEnvelope_v1 struct to a JSON string representation, +""" envelope_to_json - Convert msg_envelope_v1 to JSON string +This function converts the msg_envelope_v1 struct to a JSON string representation, preserving all metadata and payload information for NATS message publishing. # Function Workflow: -1. Creates a dictionary with envelope metadata (correlationId, msgId, timestamp, etc.) + 1. Creates a dictionary with envelope metadata (correlation_id, msg_id, timestamp, etc.) 2. Conditionally includes metadata dictionary if not empty 3. Iterates through payloads and converts each to JSON-compatible dictionary 4. Handles direct transport payloads (Base64 encoding) and link transport payloads (URL) 5. Returns final JSON string representation # Arguments: - - `env::msgEnvelope_v1` - The msgEnvelope_v1 struct to convert to JSON + - `env::msg_envelope_v1` - The msg_envelope_v1 struct to convert to JSON # Return: - `String` - JSON string representation of the envelope @@ -257,27 +256,27 @@ preserving all metadata and payload information for NATS message publishing. using UUIDs # Create an envelope with payloads -payload = msgPayload_v1("Hello", "text"; dataname="msg", transport="direct", encoding="base64") -env = msgEnvelope_v1("my.subject", [payload]) +payload = msg_payload_v1("Hello", "text"; dataname="msg", transport="direct", encoding="base64") +env = msg_envelope_v1("my.subject", [payload]) # Convert to JSON for publishing json_msg = envelope_to_json(env) ``` """ -function envelope_to_json(env::msgEnvelope_v1) +function envelope_to_json(env::msg_envelope_v1) obj = Dict{String, Any}( - "correlationId" => env.correlationId, - "msgId" => env.msgId, + "correlation_id" => env.correlation_id, + "msg_id" => env.msg_id, "timestamp" => env.timestamp, - "sendTo" => env.sendTo, - "msgPurpose" => env.msgPurpose, - "senderName" => env.senderName, - "senderId" => env.senderId, - "receiverName" => env.receiverName, - "receiverId" => env.receiverId, - "replyTo" => env.replyTo, - "replyToMsgId" => env.replyToMsgId, - "brokerURL" => env.brokerURL + "send_to" => env.send_to, + "msg_purpose" => env.msg_purpose, + "sender_name" => env.sender_name, + "sender_id" => env.sender_id, + "receiver_name" => env.receiver_name, + "receiver_id" => env.receiver_id, + "reply_to" => env.reply_to, + "reply_to_msg_id" => env.reply_to_msg_id, + "broker_url" => env.broker_url ) if !isempty(env.metadata) # Only include metadata if it exists and is not empty @@ -291,7 +290,7 @@ function envelope_to_json(env::msgEnvelope_v1) payload_obj = Dict{String, Any}( "id" => payload.id, "dataname" => payload.dataname, - "type" => payload.type, + "payload_type" => payload.type, "transport" => payload.transport, "encoding" => payload.encoding, "size" => payload.size, @@ -359,8 +358,8 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c 1. Iterates through the list of (dataname, data, type) tuples 2. For each payload: extracts the type from the tuple and serializes accordingly 3. Compares the serialized size against `size_threshold` -4. For small payloads: encodes as Base64, constructs a "direct" msgPayload_v1 -5. For large payloads: uploads to the fileserver, constructs a "link" msgPayload_v1 with the URL +4. For small payloads: encodes as Base64, constructs a "direct" msg_payload_v1 +5. For large payloads: uploads to the fileserver, constructs a "link" msg_payload_v1 with the URL 6. Converts envelope to JSON string and optionally publishes to NATS # Arguments: @@ -368,13 +367,13 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c - `data::AbstractArray{Tuple{String, Any, String}}` - List of (dataname, data, type) tuples to send - `dataname::String` - Name of the payload - `data::Any` - The actual data to send - - `type::String` - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" + - `payload_type::String` - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" - No standalone `type` parameter - type is specified per payload # Keyword Arguments: - - `nats_url::String = DEFAULT_NATS_URL` - URL of the NATS server + - `broker_url::String = DEFAULT_BROKER_URL` - URL of the NATS server - `fileserver_url = DEFAULT_FILESERVER_URL` - URL of the HTTP file server for large payloads - - `fileserverUploadHandler::Function = plik_oneshot_upload` - Function to handle fileserver uploads (must return Dict with "status", "uploadid", "fileid", "url" keys) + - `fileserver_upload_handler::Function = plik_oneshot_upload` - Function to handle fileserver uploads (must return Dict with "status", "uploadid", "fileid", "url" keys) - `size_threshold::Int = DEFAULT_SIZE_THRESHOLD` - Threshold in bytes separating direct vs link transport - `correlation_id::Union{String, Nothing} = nothing` - Optional correlation ID for tracing; if `nothing`, a UUID is generated - `msg_purpose::String = "chat"` - Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", "chat", etc. @@ -387,7 +386,7 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c # Return: - A tuple `(env, env_json_str)` where: - - `env::msgEnvelope_v1` - The envelope object containing all metadata and payloads + - `env::msg_envelope_v1` - The envelope object containing all metadata and payloads - `env_json_str::String` - JSON string representation of the envelope for publishing # Example @@ -415,15 +414,15 @@ env, msg_json = smartsend("chat.subject", [ ]) # Publish the JSON string directly using NATS request-reply pattern -# reply = NATS.request(nats_url, subject, env_json_str; reply_to=reply_to_topic) +# reply = NATS.request(broker_url, subject, env_json_str; reply_to=reply_to_topic) ``` -""" #[PENDING] +""" function smartsend( subject::String, # smartreceive's subject data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples. Use Tuple{String, Any, String}[] for empty payloads - nats_url::String = DEFAULT_NATS_URL, + broker_url::String = DEFAULT_BROKER_URL, # NATS server URL fileserver_url = DEFAULT_FILESERVER_URL, - fileserverUploadHandler::Function=plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver + fileserver_upload_handler::Function = plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver size_threshold::Int = DEFAULT_SIZE_THRESHOLD, correlation_id::Union{String, Nothing} = nothing, msg_purpose::String = "chat", @@ -444,13 +443,13 @@ function smartsend( msg_id = string(uuid4()) # Process each payload in the list - payloads = msgPayload_v1[] + payloads = msg_payload_v1[] for (dataname, payload_data, payload_type) in data # Serialize data based on type payload_bytes = _serialize_data(payload_data, payload_type) payload_size = length(payload_bytes) # Calculate payload size in bytes - log_trace(cid, "Serialized payload '$dataname' (type: $payload_type) size: $payload_size bytes") # Log payload size + log_trace(cid, "Serialized payload '$dataname' (payload_type: $payload_type) size: $payload_size bytes") # Log payload size # Decision: Direct vs Link if payload_size < size_threshold # Check if payload is small enough for direct transport @@ -458,8 +457,8 @@ function smartsend( payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string log_trace(cid, "Using direct transport for $payload_size bytes") # Log transport choice - # Create msgPayload_v1 for direct transport - payload = msgPayload_v1( + # Create msg_payload_v1 for direct transport + payload = msg_payload_v1( payload_b64, payload_type; id = string(uuid4()), @@ -474,18 +473,18 @@ function smartsend( # Link path - Upload to HTTP server, send URL via NATS log_trace(cid, "Using link transport, uploading to fileserver") # Log link transport choice - # Upload to HTTP server - response = fileserverUploadHandler(fileserver_url, dataname, payload_bytes) + # Upload to HTTP server + response = fileserver_upload_handler(fileserver_url, dataname, payload_bytes) if response["status"] != 200 # Check if upload was successful - error("Failed to upload data to fileserver: $(response["status"])") # Throw error if upload failed + error("Failed to upload data to fileserver: $(response["status"])") # Throw error if upload failed end url = response["url"] # URL for the uploaded data log_trace(cid, "Uploaded to URL: $url") # Log successful upload - # Create msgPayload_v1 for link transport - payload = msgPayload_v1( + # Create msg_payload_v1 for link transport + payload = msg_payload_v1( url, payload_type; id = string(uuid4()), @@ -499,26 +498,26 @@ function smartsend( end end - # Create msgEnvelope_v1 with all payloads - env = msgEnvelope_v1( - subject, - payloads; - correlationId = cid, - msgId = msg_id, - msgPurpose = msg_purpose, - senderName = sender_name, - senderId = string(uuid4()), - receiverName = receiver_name, - receiverId = receiver_id, - replyTo = reply_to, - replyToMsgId = reply_to_msg_id, - brokerURL = nats_url, - metadata = Dict{String, Any}(), - ) + # Create msg_envelope_v1 with all payloads + env = msg_envelope_v1( + subject, + payloads; + correlation_id = cid, + msg_id = msg_id, + msg_purpose = msg_purpose, + sender_name = sender_name, + sender_id = string(uuid4()), + receiver_name = receiver_name, + receiver_id = receiver_id, + reply_to = reply_to, + reply_to_msg_id = reply_to_msg_id, + broker_url = broker_url, + metadata = Dict{String, Any}(), + ) env_json_str = envelope_to_json(env) # Convert envelope to JSON if is_publish - publish_message(nats_url, subject, env_json_str, cid) # Publish message to NATS + publish_message(broker_url, subject, env_json_str, cid) # Publish message to NATS end return (env, env_json_str) @@ -539,7 +538,7 @@ It supports multiple serialization formats for different data types. # Arguments: - `data::Any` - Data to serialize (string for `"text"`, JSON-serializable for `"dictionary"`, table-like for `"table"`, binary for `"image"`, `"audio"`, `"video"`, `"binary"`) - - `type::String` - Target format: "text", "dictionary", "table", "image", "audio", "video", "binary" + - `payload_type::String` - Target format: "text", "dictionary", "table", "image", "audio", "video", "binary" # Return: - `Vector{UInt8}` - Binary representation of the serialized data @@ -585,7 +584,7 @@ binary_bytes = _serialize_data(buf, "binary") binary_bytes_direct = _serialize_data(UInt8[1, 2, 3], "binary") ``` """ -function _serialize_data(data::Any, type::String) +function _serialize_data(data::Any, payload_type::String) """ Example on how JSON.jl convert: dictionary -> json string -> json string bytes -> json string -> json object d = Dict( "name"=>"ton", @@ -602,40 +601,40 @@ function _serialize_data(data::Any, type::String) json_obj = JSON.parse(json_str_2) """ - if type == "text" # Text data - convert to UTF-8 bytes + if payload_type == "text" # Text data - convert to UTF-8 bytes if isa(data, String) data_bytes = Vector{UInt8}(data) # Convert string to UTF-8 bytes return data_bytes else error("Text data must be a String") end - elseif type == "dictionary" # JSON data - serialize directly + elseif payload_type == "dictionary" # JSON data - serialize directly json_str = JSON.json(data) # Convert Julia data to JSON string json_str_bytes = Vector{UInt8}(json_str) # Convert JSON string to bytes return json_str_bytes - elseif type == "table" # Table data - convert to Arrow IPC stream + elseif payload_type == "table" # Table data - convert to Arrow IPC stream io = IOBuffer() # Create in-memory buffer Arrow.write(io, data) # Write data as Arrow IPC stream to buffer return take!(io) # Return the buffer contents as bytes - elseif type == "image" # Image data - treat as binary + elseif payload_type == "image" # Image data - treat as binary if isa(data, Vector{UInt8}) return data # Return binary data directly else error("Image data must be Vector{UInt8}") end - elseif type == "audio" # Audio data - treat as binary + elseif payload_type == "audio" # Audio data - treat as binary if isa(data, Vector{UInt8}) return data # Return binary data directly else error("Audio data must be Vector{UInt8}") end - elseif type == "video" # Video data - treat as binary + elseif payload_type == "video" # Video data - treat as binary if isa(data, Vector{UInt8}) return data # Return binary data directly else error("Video data must be Vector{UInt8}") end - elseif type == "binary" # Binary data - treat as binary + elseif payload_type == "binary" # Binary data - treat as binary if isa(data, IOBuffer) # Check if data is an IOBuffer return take!(data) # Return buffer contents as bytes elseif isa(data, Vector{UInt8}) # Check if data is already binary @@ -644,7 +643,7 @@ function _serialize_data(data::Any, type::String) error("Binary data must be binary (Vector{UInt8} or IOBuffer)") end else # Unknown type - error("Unknown type: $type") + error("Unknown payload_type: $type") end end @@ -654,7 +653,7 @@ This internal function publishes a message to a NATS subject with proper connection management and logging. # Arguments: - - `nats_url::String` - NATS server URL (e.g., "nats://localhost:4222") + - `broker_url::String` - NATS server URL (e.g., "nats://localhost:4222") - `subject::String` - NATS subject to publish to (e.g., "/agent/wine/api/v1/prompt") - `message::String` - JSON message to publish - `correlation_id::String` - Correlation ID for tracing and logging @@ -663,18 +662,18 @@ connection management and logging. - `nothing` - This function performs publishing but returns nothing # Example -```jldoctest -using NATS + ```jldoctest + using NATS -# Prepare JSON message -message = "{\"correlationId\":\"abc123\",\"payload\":\"test\"}" + # Prepare JSON message + message = "{\"correlation_id\":\"abc123\",\"payload\":\"test\"}" -# Publish to NATS -publish_message("nats://localhost:4222", "my.subject", message, "abc123") -``` + # Publish to NATS + publish_message("nats://localhost:4222", "my.subject", message, "abc123") + ``` """ -function publish_message(nats_url::String, subject::String, message::String, correlation_id::String) - conn = NATS.connect(nats_url) # Create NATS connection +function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) + conn = NATS.connect(broker_url) # Create NATS connection try NATS.publish(conn, subject, message) # Publish message to NATS log_trace(correlation_id, "Message published to $subject") # Log successful publish @@ -701,7 +700,7 @@ A HTTP file server is required along with its download function. - `msg::NATS.Msg` - NATS message to process # Keyword Arguments: - - `fileserverDownloadHandler::Function = _fetch_with_backoff` - Function to handle downloading data from file server URLs + - `fileserver_download_handler::Function = _fetch_with_backoff` - Function to handle downloading data from file server URLs - `max_retries::Int = 5` - Maximum retry attempts for fetching URL - `base_delay::Int = 100` - Initial delay for exponential backoff in ms - `max_delay::Int = 5000` - Maximum delay for exponential backoff in ms @@ -710,23 +709,23 @@ A HTTP file server is required along with its download function. - `AbstractArray{Tuple{String, Any, String}}` - List of (dataname, data, type) tuples # Example -```jldoctest -# Receive and process message -msg = nats_message # NATS message -payloads = smartreceive(msg; fileserverDownloadHandler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) -# payloads = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] -``` + ```jldoctest + # Receive and process message + msg = nats_message # NATS message + payloads = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) + # payloads = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] + ``` """ function smartreceive( msg::NATS.Msg; - fileserverDownloadHandler::Function=_fetch_with_backoff, + fileserver_download_handler::Function=_fetch_with_backoff, max_retries::Int = 5, base_delay::Int = 100, max_delay::Int = 5000 ) # Parse the JSON envelope - json_data = JSON.parse(String(msg.payload)) - log_trace(json_data["correlationId"], "Processing received message") # Log message processing start + json_data = JSON.parse(String(msg.payload)) + log_trace(json_data["correlation_id"], "Processing received message") # Log message processing start # Process all payloads in the envelope payloads_list = Tuple{String, Any, String}[] @@ -740,7 +739,7 @@ function smartreceive( dataname = String(payload["dataname"]) if transport == "direct" # Direct transport - payload is in the message - log_trace(json_data["correlationId"], "Direct transport - decoding payload '$dataname'") # Log direct transport handling + log_trace(json_data["correlation_id"], "Direct transport - decoding payload '$dataname'") # Log direct transport handling # Extract base64 payload from the payload payload_b64 = String(payload["data"]) @@ -749,21 +748,21 @@ function smartreceive( payload_bytes = Base64.base64decode(payload_b64) # Decode base64 payload to bytes # Deserialize based on type - data_type = String(payload["type"]) - data = _deserialize_data(payload_bytes, data_type, json_data["correlationId"]) + data_type = String(payload["payload_type"]) + data = _deserialize_data(payload_bytes, data_type, json_data["correlation_id"]) push!(payloads_list, (dataname, data, data_type)) elseif transport == "link" # Link transport - payload is at URL # Extract download URL from the payload url = String(payload["data"]) - log_trace(json_data["correlationId"], "Link transport - fetching '$dataname' from URL: $url") # Log link transport handling + log_trace(json_data["correlation_id"], "Link transport - fetching '$dataname' from URL: $url") # Log link transport handling # Fetch with exponential backoff using the download handler - downloaded_data = fileserverDownloadHandler(url, max_retries, base_delay, max_delay, json_data["correlationId"]) + downloaded_data = fileserver_download_handler(url, max_retries, base_delay, max_delay, json_data["correlation_id"]) # Deserialize based on type - data_type = String(payload["type"]) - data = _deserialize_data(downloaded_data, data_type, json_data["correlationId"]) + data_type = String(payload["payload_type"]) + data = _deserialize_data(downloaded_data, data_type, json_data["correlation_id"]) push!(payloads_list, (dataname, data, data_type)) else # Unknown transport type @@ -851,7 +850,7 @@ It handles "text" (string), "dictionary" (JSON deserialization), "table" (Arrow # Arguments: - `data::Vector{UInt8}` - Serialized data as bytes - - `type::String` - Data type ("text", "dictionary", "table", "image", "audio", "video", "binary") + - `payload_type::String` - Data type ("text", "dictionary", "table", "image", "audio", "video", "binary") - `correlation_id::String` - Correlation ID for logging # Return: @@ -877,28 +876,28 @@ table_data = _deserialize_data(arrow_bytes, "table", "correlation123") """ function _deserialize_data( data::Vector{UInt8}, - type::String, + payload_type::String, correlation_id::String ) - if type == "text" # Text data - convert to string + if payload_type == "text" # Text data - convert to string return String(data) # Convert bytes to string - elseif type == "dictionary" # JSON data - deserialize + elseif payload_type == "dictionary" # JSON data - deserialize json_str = String(data) # Convert bytes to string return JSON.parse(json_str) # Parse JSON string to JSON object - elseif type == "table" # Table data - deserialize Arrow IPC stream + elseif payload_type == "table" # Table data - deserialize Arrow IPC stream io = IOBuffer(data) # Create buffer from bytes df = Arrow.Table(io) # Read Arrow IPC format from buffer return df # Return DataFrame - elseif type == "image" # Image data - return binary + elseif payload_type == "image" # Image data - return binary return data # Return bytes directly - elseif type == "audio" # Audio data - return binary + elseif payload_type == "audio" # Audio data - return binary return data # Return bytes directly - elseif type == "video" # Video data - return binary + elseif payload_type == "video" # Video data - return binary return data # Return bytes directly - elseif type == "binary" # Binary data - return binary + elseif payload_type == "binary" # Binary data - return binary return data # Return bytes directly else # Unknown type - error("Unknown type: $type") # Throw error for unknown type + error("Unknown payload_type: $payload_type") # Throw error for unknown type end end @@ -915,9 +914,9 @@ retrieves an upload ID and token, then uploads the file data as multipart form d 4. Returns identifiers and download URL for the uploaded file # Arguments: - - `fileServerURL::String` - Base URL of the plik server (e.g., `"http://localhost:8080"`) - - `filename::String` - Name of the file being uploaded - - `data::Vector{UInt8}` - Raw byte data of the file content + - `file_server_url::String` - Base URL of the plik server (e.g., `"http://localhost:8080"`) + - `filename::String` - Name of the file being uploaded + - `data::Vector{UInt8}` - Raw byte data of the file content # Return: - `Dict{String, Any}` - Dictionary with keys: @@ -927,36 +926,36 @@ retrieves an upload ID and token, then uploads the file data as multipart form d - `"url"` - Full URL to download the uploaded file # Example -```jldoctest -using HTTP, JSON + ```jldoctest + using HTTP, JSON -fileServerURL = "http://localhost:8080" -filename = "test.txt" -data = UInt8["hello world"] + file_server_url = "http://localhost:8080" + filename = "test.txt" + data = UInt8["hello world"] -# Upload to local plik server -result = plik_oneshot_upload(fileServerURL, filename, data) + # Upload to local plik server + result = plik_oneshot_upload(file_server_url, filename, data) -# Access the result as a Dict -# result["status"], result["uploadid"], result["fileid"], result["url"] -``` + # Access the result as a Dict + # result["status"], result["uploadid"], result["fileid"], result["url"] + ``` """ -function plik_oneshot_upload(fileServerURL::String, filename::String, data::Vector{UInt8}) +function plik_oneshot_upload(file_server_url::String, filename::String, data::Vector{UInt8}) # ----------------------------------------- get upload id ---------------------------------------- # # Equivalent curl command: curl -X POST -d '{ "OneShot" : true }' http://localhost:8080/upload - url_getUploadID = "$fileServerURL/upload" # URL to get upload ID + url_getUploadID = "$file_server_url/upload" # URL to get upload ID headers = ["Content-Type" => "application/json"] body = """{ "OneShot" : true }""" - httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) - responseJson = JSON.parse(httpResponse.body) - uploadid = responseJson["id"] - uploadtoken = responseJson["uploadToken"] + http_response = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) + response_json = JSON.parse(http_response.body) + uploadid = response_json["id"] + uploadtoken = response_json["uploadToken"] # ------------------------------------------ upload file ----------------------------------------- # # Equivalent curl command: curl -X POST --header "X-UploadToken: UPLOAD_TOKEN" -F "file=@PATH_TO_FILE" http://localhost:8080/file/UPLOAD_ID file_multipart = HTTP.Multipart(filename, IOBuffer(data), "application/octet-stream") # Plik won't accept raw bytes upload - url_upload = "$fileServerURL/file/$uploadid" + url_upload = "$file_server_url/file/$uploadid" headers = ["X-UploadToken" => uploadtoken] # Create the multipart form data @@ -965,24 +964,24 @@ function plik_oneshot_upload(fileServerURL::String, filename::String, data::Vect )) # Execute the POST request - httpResponse = nothing + http_response = nothing try - httpResponse = HTTP.post(url_upload, headers, form) - responseJson = JSON.parse(httpResponse.body) + http_response = HTTP.post(url_upload, headers, form) + response_json = JSON.parse(http_response.body) catch e @error "Request failed" exception=e end - fileid = responseJson["id"] + fileid = response_json["id"] # url of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" - url = "$fileServerURL/file/$uploadid/$fileid/$filename" + url = "$file_server_url/file/$uploadid/$fileid/$filename" - return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) + return Dict("status" => http_response.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) end -""" plik_oneshot_upload(fileServerURL::String, filepath::String) +""" plik_oneshot_upload(file_server_url::String, filepath::String) This function uploads a file from disk to a plik server in one-shot mode (no upload session). It first creates a one-shot upload session by sending a POST request with `{"OneShot": true}`, retrieves an upload ID and token, then uploads the file data as multipart form data using the token. @@ -994,7 +993,7 @@ retrieves an upload ID and token, then uploads the file data as multipart form d 4. Returns identifiers and download URL for the uploaded file # Arguments: - - `fileServerURL::String` - Base URL of the plik server (e.g., `"http://localhost:8080"`) + - `file_server_url::String` - Base URL of the plik server (e.g., `"http://localhost:8080"`) - `filepath::String` - Full path to the local file to upload # Return: @@ -1008,56 +1007,54 @@ retrieves an upload ID and token, then uploads the file data as multipart form d ```jldoctest using HTTP, JSON -fileServerURL = "http://localhost:8080" +file_server_url = "http://localhost:8080" filepath = "./test.zip" # Upload to local plik server -result = plik_oneshot_upload(fileServerURL, filepath) +result = plik_oneshot_upload(file_server_url, filepath) # Access the result as a Dict # result["status"], result["uploadid"], result["fileid"], result["url"] ``` """ -function plik_oneshot_upload(fileServerURL::String, filepath::String) +function plik_oneshot_upload(file_server_url::String, filepath::String) # ----------------------------------------- get upload id ---------------------------------------- # # Equivalent curl command: curl -X POST -d '{ "OneShot" : true }' http://localhost:8080/upload filename = basename(filepath) - url_getUploadID = "$fileServerURL/upload" # URL to get upload ID + url_getUploadID = "$file_server_url/upload" # URL to get upload ID headers = ["Content-Type" => "application/json"] body = """{ "OneShot" : true }""" - httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) - responseJson = JSON.parse(httpResponse.body) + http_response = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false) + response_json = JSON.parse(http_response.body) - uploadid = responseJson["id"] - uploadtoken = responseJson["uploadToken"] + uploadid = response_json["id"] + uploadtoken = response_json["uploadToken"] # ------------------------------------------ upload file ----------------------------------------- # # Equivalent curl command: curl -X POST --header "X-UploadToken: UPLOAD_TOKEN" -F "file=@PATH_TO_FILE" http://localhost:8080/file/UPLOAD_ID - file_multipart = open(filepath, "r") - url_upload = "$fileServerURL/file/$uploadid" + url_upload = "$file_server_url/file/$uploadid" headers = ["X-UploadToken" => uploadtoken] + response = open(filepath, "r") do file_stream + form = HTTP.Form(Dict("file" => file_stream)) + + # Adding status_exception=false prevents 4xx/5xx from triggering 'catch' + HTTP.post(url_upload, headers, form; status_exception = false) + end - # Create the multipart form data - form = HTTP.Form(Dict( - "file" => file_multipart - )) - - # Execute the POST request - httpResponse = nothing - try - httpResponse = HTTP.post(url_upload, headers, form) - responseJson = JSON.parse(httpResponse.body) - catch e - @error "Request failed" exception=e + if !isnothing(response) && response.status == 200 + println("Success!") + else + println("Server returned an error code: ", response.status) end + response_json = JSON.parse(response.body) - fileid = responseJson["id"] + fileid = response_json["id"] # url of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" - url = "$fileServerURL/file/$uploadid/$fileid/$filename" + url = "$file_server_url/file/$uploadid/$fileid/$filename" - return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) + return Dict("status" => http_response.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) end -- 2.49.1 From 3e2b8b1e3a8d9c5711bdeef4d4b316e80b6ad286 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 18:19:03 +0700 Subject: [PATCH 12/35] update --- src/NATSBridge.jl | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 4677c53..a5a7faf 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -290,7 +290,7 @@ function envelope_to_json(env::msg_envelope_v1) payload_obj = Dict{String, Any}( "id" => payload.id, "dataname" => payload.dataname, - "payload_type" => payload.type, + "payload_type" => payload.payload_type, "transport" => payload.transport, "encoding" => payload.encoding, "size" => payload.size, @@ -643,7 +643,7 @@ function _serialize_data(data::Any, payload_type::String) error("Binary data must be binary (Vector{UInt8} or IOBuffer)") end else # Unknown type - error("Unknown payload_type: $type") + error("Unknown payload_type: $payload_type") end end @@ -967,11 +967,10 @@ function plik_oneshot_upload(file_server_url::String, filename::String, data::Ve http_response = nothing try http_response = HTTP.post(url_upload, headers, form) - response_json = JSON.parse(http_response.body) catch e - @error "Request failed" exception=e + @error "Request failed" exception=e end - + response_json = JSON.parse(http_response.body) fileid = response_json["id"] # url of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" @@ -1035,19 +1034,19 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) # Equivalent curl command: curl -X POST --header "X-UploadToken: UPLOAD_TOKEN" -F "file=@PATH_TO_FILE" http://localhost:8080/file/UPLOAD_ID url_upload = "$file_server_url/file/$uploadid" headers = ["X-UploadToken" => uploadtoken] - response = open(filepath, "r") do file_stream + http_response = open(filepath, "r") do file_stream form = HTTP.Form(Dict("file" => file_stream)) # Adding status_exception=false prevents 4xx/5xx from triggering 'catch' HTTP.post(url_upload, headers, form; status_exception = false) end - if !isnothing(response) && response.status == 200 + if !isnothing(http_response) && http_response.status == 200 println("Success!") else - println("Server returned an error code: ", response.status) + println("Server returned an error code: ", http_response.status) end - response_json = JSON.parse(response.body) + response_json = JSON.parse(http_response.body) fileid = response_json["id"] @@ -1057,6 +1056,10 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) return Dict("status" => http_response.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) end +function _get_payload_bytes(data) + @error "did't implement yet" +end + @@ -1074,6 +1077,4 @@ end - - end # module -- 2.49.1 From 45f125789674b9daf523674ca189fa582fce91a0 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 18:50:28 +0700 Subject: [PATCH 13/35] update --- src/NATSBridge.jl | 64 +++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index a5a7faf..4edc503 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -100,9 +100,9 @@ payload = msg_payload_v1( struct msg_payload_v1 id::String # id of this payload e.g. "uuid4" dataname::String # name of this payload e.g. "login_image" - payload_type::String # this payload type. Can be "text | dictionary | table | image | audio | video | binary" - transport::String # "direct | link" - encoding::String # "none | json | base64 | arrow-ipc" + payload_type::String # this payload type. Can be "text", "dictionary", "table", "image", "audio", "video", "binary" + transport::String # transport method: "direct" or "link" + encoding::String # encoding method: "none", "json", "base64", "arrow-ipc" size::Integer # data size in bytes e.g. 15433 data::Any # payload data in case of direct transport or a URL in case of link metadata::Dict{String, Any} # Dict("checksum" => "sha256_hash", ...) This metadata is for this payload @@ -138,7 +138,7 @@ with metadata for routing, tracing, and message context. # Arguments: - `send_to::String` - NATS subject/topic to publish the message to (e.g., "/agent/wine/api/v1/prompt") - - `payloads::AbstractArray{msg_payload_v1}` - List of payloads to include in the message + - `payloads::Vector{msg_payload_v1}` - List of payloads to include in the message # Keyword Arguments: - `correlation_id::String = ""` - Unique identifier to track messages across systems; auto-generated if empty @@ -180,27 +180,27 @@ env = msg_envelope_v1( struct msg_envelope_v1 correlation_id::String # Unique identifier to track messages across systems. Many senders can talk about the same topic. msg_id::String # this message id - timestamp::String # message published timestamp. string(Dates.now()) - + timestamp::String # message published timestamp (string(Dates.now())) + send_to::String # topic/subject the sender sends to e.g. "/agent/wine/api/v1/prompt" - msg_purpose::String # purpose of this message e.g. "ACK | NACK | updateStatus | shutdown | ..." + msg_purpose::String # purpose of this message e.g. "ACK", "NACK", "updateStatus", "shutdown", ... sender_name::String # sender name (String) e.g. "agent-wine-web-frontend" - sender_id::String # sender id e.g. uuid4snakecase() + sender_id::String # sender id e.g. uuid4() receiver_name::String # msg receiver name (String) e.g. "agent-backend" - receiver_id::String # msg receiver id, nothing means everyone in the topic e.g. uuid4snakecase() - + receiver_id::String # msg receiver id, nothing means everyone in the topic e.g. uuid4() + reply_to::String # sender ask receiver to reply to this topic - reply_to_msg_id::String # the message id this message is replying to - broker_url::String # mqtt/NATS server address - + reply_to_msg_id::String # the message id this message is replying to + broker_url::String # NATS server address + metadata::Dict{String, Any} - payloads::AbstractArray{msg_payload_v1} # multiple payload store here + payloads::Vector{msg_payload_v1} # multiple payload store here end # constructor function msg_envelope_v1( send_to::String, - payloads::AbstractArray{msg_payload_v1}; + payloads::Vector{msg_payload_v1}; correlation_id::String = "", msg_id::String = "", timestamp::String = string(Dates.now()), @@ -422,7 +422,7 @@ function smartsend( data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples. Use Tuple{String, Any, String}[] for empty payloads broker_url::String = DEFAULT_BROKER_URL, # NATS server URL fileserver_url = DEFAULT_FILESERVER_URL, - fileserver_upload_handler::Function = plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver + fileserver_upload_handler::Function = plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver size_threshold::Int = DEFAULT_SIZE_THRESHOLD, correlation_id::Union{String, Nothing} = nothing, msg_purpose::String = "chat", @@ -544,8 +544,8 @@ It supports multiple serialization formats for different data types. - `Vector{UInt8}` - Binary representation of the serialized data # Throws: - - `Error` if `type` is not one of the supported types - - `Error` if `type` is `"image"`, `"audio"`, or `"video"` but `data` is not `Vector{UInt8}` + - `Error` if `payload_type` is not one of the supported types + - `Error` if `payload_type` is `"image"`, `"audio"`, or `"video"` but `data` is not `Vector{UInt8}` # Example ```jldoctest @@ -706,15 +706,15 @@ A HTTP file server is required along with its download function. - `max_delay::Int = 5000` - Maximum delay for exponential backoff in ms # Return: - - `AbstractArray{Tuple{String, Any, String}}` - List of (dataname, data, type) tuples + - `Vector{Tuple{String, Any, String}}` - List of (dataname, data, type) tuples # Example - ```jldoctest - # Receive and process message - msg = nats_message # NATS message - payloads = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) - # payloads = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] - ``` +```jldoctest +# Receive and process message +msg = nats_message # NATS message +payloads = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) +# payloads = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] +``` """ function smartreceive( msg::NATS.Msg; @@ -857,12 +857,12 @@ It handles "text" (string), "dictionary" (JSON deserialization), "table" (Arrow - Deserialized data (String for "text", DataFrame for "table", JSON data for "dictionary", bytes for "image", "audio", "video", "binary") # Throws: - - `Error` if `type` is not one of the supported types + - `Error` if `payload_type` is not one of the supported types # Example ```jldoctest # Text data -text_bytes = UInt8["Hello World"] +text_bytes = Vector{UInt8}("Hello World") text_data = _deserialize_data(text_bytes, "text", "correlation123") # JSON data @@ -870,7 +870,7 @@ json_bytes = UInt8[123, 34, 110, 97, 109, 101, 34, 58, 34, 65, 108, 105, 99, 101 json_data = _deserialize_data(json_bytes, "dictionary", "correlation123") # Arrow IPC data (table) -arrow_bytes = UInt8[1, 2, 3] # Arrow IPC bytes +arrow_bytes = Vector{UInt8}([1, 2, 3]) # Arrow IPC bytes table_data = _deserialize_data(arrow_bytes, "table", "correlation123") ``` """ @@ -931,7 +931,7 @@ retrieves an upload ID and token, then uploads the file data as multipart form d file_server_url = "http://localhost:8080" filename = "test.txt" - data = UInt8["hello world"] + data = Vector{UInt8}("hello world") # Upload to local plik server result = plik_oneshot_upload(file_server_url, filename, data) @@ -1042,9 +1042,9 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) end if !isnothing(http_response) && http_response.status == 200 - println("Success!") + # Success - response already logged by caller else - println("Server returned an error code: ", http_response.status) + error("Failed to upload file: server returned status $(http_response.status)") end response_json = JSON.parse(http_response.body) @@ -1057,7 +1057,7 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) end function _get_payload_bytes(data) - @error "did't implement yet" + @error "didn't implement yet" end -- 2.49.1 From b51641dc7e867c4b4ddea73333f3a2966ce6d8d9 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 20:09:10 +0700 Subject: [PATCH 14/35] update --- docs/architecture.md | 150 ++++++----- docs/implementation.md | 93 +++---- examples/tutorial.md | 2 +- src/NATSBridge.jl | 351 ++++++++++++------------- test/test_julia_dict_sender.jl | 6 +- test/test_julia_file_sender.jl | 6 +- test/test_julia_mix_payloads_sender.jl | 6 +- test/test_julia_table_sender.jl | 6 +- test/test_julia_text_sender.jl | 6 +- 9 files changed, 317 insertions(+), 309 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index fee3cb2..31e5c99 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,16 +17,16 @@ The system uses **handler functions** to abstract file server operations, allowi ```julia # Upload handler - uploads data to file server and returns URL -# The handler is passed to smartsend as fileserverUploadHandler parameter -# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8}) +# The handler is passed to smartsend as fileserver_upload_handler parameter +# It receives: (file_server_url::String, dataname::String, data::Vector{UInt8}) # Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url" -fileserverUploadHandler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} +fileserver_upload_handler(file_server_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} # Download handler - fetches data from file server URL with exponential backoff -# The handler is passed to smartreceive as fileserverDownloadHandler parameter +# The handler is passed to smartreceive as fileserver_download_handler parameter # It receives: (url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String) # Returns: Vector{UInt8} (the downloaded data) -fileserverDownloadHandler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} +fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} ``` This design allows the system to support multiple file server backends without changing the core messaging logic. @@ -40,21 +40,21 @@ The system uses a **standardized list-of-tuples format** for all payload operati # Input format for smartsend (always a list of tuples with type info) [(dataname1, data1, type1), (dataname2, data2, type2), ...] -# Output format for smartreceive (returns envelope dictionary with payloads field) -# Returns: Dict with envelope metadata and payloads field containing list of tuples +# Output format for smartreceive (returns a dictionary with payloads field containing list of tuples) +# Returns: Dict with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}} # { -# "correlationId": "...", -# "msgId": "...", +# "correlation_id": "...", +# "msg_id": "...", # "timestamp": "...", -# "sendTo": "...", -# "msgPurpose": "...", -# "senderName": "...", -# "senderId": "...", -# "receiverName": "...", -# "receiverId": "...", -# "replyTo": "...", -# "replyToMsgId": "...", -# "brokerURL": "...", +# "send_to": "...", +# "msg_purpose": "...", +# "sender_name": "...", +# "sender_id": "...", +# "receiver_name": "...", +# "receiver_id": "...", +# "reply_to": "...", +# "reply_to_msg_id": "...", +# "broker_url": "...", # "metadata": {...}, # "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] # } @@ -78,17 +78,16 @@ This design allows per-payload type specification, enabling **mixed-content mess smartsend( "/test", [("dataname1", data1, "dictionary")], # List with one tuple (data, type) - nats_url="nats://localhost:4222", - fileserverUploadHandler=plik_oneshot_upload, - metadata=user_provided_envelope_level_metadata + broker_url="nats://localhost:4222", + fileserver_upload_handler=plik_oneshot_upload ) # Multiple payloads in one message with different types smartsend( "/test", [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")], - nats_url="nats://localhost:4222", - fileserverUploadHandler=plik_oneshot_upload + broker_url="nats://localhost:4222", + fileserver_upload_handler=plik_oneshot_upload ) # Mixed content (e.g., chat with text, image, audio) @@ -99,13 +98,14 @@ smartsend( ("user_image", image_data, "image"), ("audio_clip", audio_data, "audio") ], - nats_url="nats://localhost:4222" + broker_url="nats://localhost:4222" ) # Receive returns a dictionary envelope with all metadata and deserialized payloads -env = smartreceive(msg, fileserverDownloadHandler, max_retries, base_delay, max_delay) +env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) # env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...] -# env["correlationId"], env["msgId"], etc. +# env["correlation_id"], env["msg_id"], etc. +# env is a dictionary containing envelope metadata and payloads field ``` ## Architecture Diagram @@ -138,48 +138,48 @@ flowchart TD ## System Components -### 1. msgEnvelope_v1 - Message Envelope +### 1. msg_envelope_v1 - Message Envelope -The `msgEnvelope_v1` structure provides a comprehensive message format for bidirectional communication between Julia, JavaScript, and Python/Micropython applications. +The `msg_envelope_v1` structure provides a comprehensive message format for bidirectional communication between Julia, JavaScript, and Python/Micropython applications. **Julia Structure:** ```julia -struct msgEnvelope_v1 - correlationId::String # Unique identifier to track messages across systems - msgId::String # This message id - timestamp::String # Message published timestamp +struct msg_envelope_v1 + correlation_id::String # Unique identifier to track messages across systems + msg_id::String # This message id + timestamp::String # Message published timestamp - sendTo::String # Topic/subject the sender sends to - msgPurpose::String # Purpose of this message (ACK | NACK | updateStatus | shutdown | ...) - senderName::String # Sender name (e.g., "agent-wine-web-frontend") - senderId::String # Sender id (uuid4) - receiverName::String # Message receiver name (e.g., "agent-backend") - receiverId::String # Message receiver id (uuid4 or nothing for broadcast) - replyTo::String # Topic to reply to - replyToMsgId::String # Message id this message is replying to - brokerURL::String # NATS server address + send_to::String # Topic/subject the sender sends to + msg_purpose::String # Purpose of this message (ACK | NACK | updateStatus | shutdown | ...) + sender_name::String # Sender name (e.g., "agent-wine-web-frontend") + sender_id::String # Sender id (uuid4) + receiver_name::String # Message receiver name (e.g., "agent-backend") + receiver_id::String # Message receiver id (uuid4 or nothing for broadcast) + reply_to::String # Topic to reply to + reply_to_msg_id::String # Message id this message is replying to + broker_url::String # NATS server address metadata::Dict{String, Any} - payloads::AbstractArray{msgPayload_v1} # Multiple payloads stored here + payloads::Vector{msg_payload_v1} # Multiple payloads stored here end ``` **JSON Schema:** ```json { - "correlationId": "uuid-v4-string", - "msgId": "uuid-v4-string", + "correlation_id": "uuid-v4-string", + "msg_id": "uuid-v4-string", "timestamp": "2024-01-15T10:30:00Z", - "sendTo": "topic/subject", - "msgPurpose": "ACK | NACK | updateStatus | shutdown | chat", - "senderName": "agent-wine-web-frontend", - "senderId": "uuid4", - "receiverName": "agent-backend", - "receiverId": "uuid4", - "replyTo": "topic", - "replyToMsgId": "uuid4", - "brokerURL": "nats://localhost:4222", + "send_to": "topic/subject", + "msg_purpose": "ACK | NACK | updateStatus | shutdown | chat", + "sender_name": "agent-wine-web-frontend", + "sender_id": "uuid4", + "receiver_name": "agent-backend", + "receiver_id": "uuid4", + "reply_to": "topic", + "reply_to_msg_id": "uuid4", + "broker_url": "nats://localhost:4222", "metadata": { @@ -189,7 +189,7 @@ end { "id": "uuid4", "dataname": "login_image", - "type": "image", + "payload_type": "image", "transport": "direct", "encoding": "base64", "size": 15433, @@ -201,7 +201,7 @@ end { "id": "uuid4", "dataname": "large_data", - "type": "table", + "payload_type": "table", "transport": "link", "encoding": "none", "size": 524288, @@ -214,16 +214,16 @@ end } ``` -### 2. msgPayload_v1 - Payload Structure +### 2. msg_payload_v1 - Payload Structure -The `msgPayload_v1` structure provides flexible payload handling for various data types across all supported platforms. +The `msg_payload_v1` structure provides flexible payload handling for various data types across all supported platforms. **Julia Structure:** ```julia -struct msgPayload_v1 +struct msg_payload_v1 id::String # Id of this payload (e.g., "uuid4") dataname::String # Name of this payload (e.g., "login_image") - type::String # "text | dictionary | table | image | audio | video | binary" + payload_type::String # "text | dictionary | table | image | audio | video | binary" transport::String # "direct | link" encoding::String # "none | json | base64 | arrow-ipc" size::Integer # Data size in bytes @@ -383,17 +383,25 @@ graph TD ```julia function smartsend( subject::String, - data::AbstractArray{Tuple{String, Any, String}}; # No standalone type parameter - nats_url::String = "nats://localhost:4222", - fileserverUploadHandler::Function = plik_oneshot_upload, - size_threshold::Int = 1_000_000 # 1MB - is_publish::Bool = true # Whether to automatically publish to NATS + data::AbstractArray{Tuple{String, Any, String}, 1}; # List of (dataname, data, type) tuples + broker_url::String = DEFAULT_BROKER_URL, # NATS server URL + fileserver_url = DEFAULT_FILESERVER_URL, + fileserver_upload_handler::Function = plik_oneshot_upload, + size_threshold::Int = DEFAULT_SIZE_THRESHOLD, + correlation_id::Union{String, Nothing} = nothing, + msg_purpose::String = "chat", + sender_name::String = "NATSBridge", + receiver_name::String = "", + receiver_id::String = "", + reply_to::String = "", + reply_to_msg_id::String = "", + is_publish::Bool = true # Whether to automatically publish to NATS ) ``` **Return Value:** - Returns a tuple `(env, env_json_str)` where: - - `env::msgEnvelope_v1` - The envelope object containing all metadata and payloads + - `env::msg_envelope_v1` - The envelope object containing all metadata and payloads - `env_json_str::String` - JSON string representation of the envelope for publishing **Options:** @@ -417,8 +425,8 @@ The envelope object can be accessed directly for programmatic use, while the JSO ```julia function smartreceive( - msg::NATS.Message, - fileserverDownloadHandler::Function; + msg::NATS.Msg; + fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, base_delay::Int = 100, max_delay::Int = 5000 @@ -427,7 +435,7 @@ function smartreceive( # Iterate through all payloads # For each payload: check transport type # If direct: decode Base64 payload - # If link: fetch from URL with exponential backoff using fileserverDownloadHandler + # If link: fetch from URL with exponential backoff using fileserver_download_handler # Deserialize payload based on type # Return envelope dictionary with all metadata and deserialized payloads end @@ -435,7 +443,7 @@ end **Output Format:** - Returns a dictionary (key-value map) containing all envelope fields: - - `correlationId`, `msgId`, `timestamp`, `sendTo`, `msgPurpose`, `senderName`, `senderId`, `receiverName`, `receiverId`, `replyTo`, `replyToMsgId`, `brokerURL` + - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - `metadata` - Message-level metadata dictionary - `payloads` - List of dictionaries, each containing deserialized payload data @@ -445,11 +453,11 @@ end 3. For each payload: - Determine transport type (`direct` or `link`) - If `direct`: decode Base64 data from the message - - If `link`: fetch data from URL using exponential backoff (via `fileserverDownloadHandler`) + - If `link`: fetch data from URL using exponential backoff (via `fileserver_download_handler`) - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) 4. Return envelope dictionary with `payloads` field containing list of `(dataname, data, type)` tuples -**Note:** The `fileserverDownloadHandler` receives `(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)` and returns `Vector{UInt8}`. +**Note:** The `fileserver_download_handler` receives `(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)` and returns `Vector{UInt8}`. ### JavaScript Implementation diff --git a/docs/implementation.md b/docs/implementation.md index 9a02d5d..7b68f11 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -31,18 +31,18 @@ The implementation uses a **standardized list-of-tuples format** for all payload # Output format for smartreceive (returns envelope dictionary with payloads field) # Returns: Dict with envelope metadata and payloads field containing list of tuples # { -# "correlationId": "...", -# "msgId": "...", +# "correlation_id": "...", +# "msg_id": "...", # "timestamp": "...", -# "sendTo": "...", -# "msgPurpose": "...", -# "senderName": "...", -# "senderId": "...", -# "receiverName": "...", -# "receiverId": "...", -# "replyTo": "...", -# "replyToMsgId": "...", -# "brokerURL": "...", +# "send_to": "...", +# "msg_purpose": "...", +# "sender_name": "...", +# "sender_id": "...", +# "receiver_name": "...", +# "receiver_id": "...", +# "reply_to": "...", +# "reply_to_msg_id": "...", +# "broker_url": "...", # "metadata": {...}, # "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] # } @@ -53,15 +53,15 @@ Where `type` can be: `"text"`, `"dictionary"`, `"table"`, `"image"`, `"audio"`, **Examples:** ```julia # Single payload - still wrapped in a list (type is required as third element) -smartsend("/test", [(dataname1, data1, "text")], ...) +smartsend("/test", [(dataname1, data1, "text")], broker_url="nats://localhost:4222") # Multiple payloads in one message (each payload has its own type) -smartsend("/test", [(dataname1, data1, "dictionary"), (dataname2, data2, "table")], ...) +smartsend("/test", [(dataname1, data1, "dictionary"), (dataname2, data2, "table")], broker_url="nats://localhost:4222") # Receive returns a dictionary envelope with all metadata and deserialized payloads -env = smartreceive(msg, ...) +env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) # env["payloads"] = [(dataname1, data1, "text"), (dataname2, data2, "table"), ...] -# env["correlationId"], env["msgId"], etc. +# env["correlation_id"], env["msg_id"], etc. ``` ## Cross-Platform Interoperability @@ -98,7 +98,7 @@ NATSBridge is designed for seamless communication between Julia, JavaScript, and # Julia sender using NATSBridge data = [("message", "Hello from Julia!", "text")] -smartsend("/cross_platform", data, nats_url="nats://localhost:4222") +smartsend("/cross_platform", data, broker_url="nats://localhost:4222") ``` ```javascript @@ -152,8 +152,8 @@ The `smartsend` function now returns a tuple containing both the envelope object ```julia env, env_json_str = smartsend(...) -# env::msgEnvelope_v1 - The envelope object with all metadata and payloads -# env_json_str::String - JSON string for publishing to NATS +# env::msg_envelope_v1 - The envelope object with all metadata and payloads +# env_json_str::String - JSON string for publishing to NATS ``` **Options:** @@ -167,9 +167,10 @@ This enables two use cases: The Julia implementation provides: -- **[`MessageEnvelope`](src/NATSBridge.jl)**: Struct for the unified JSON envelope -- **[`SmartSend()`](src/NATSBridge.jl)**: Handles transport selection based on payload size -- **[`SmartReceive()`](src/NATSBridge.jl)**: Handles both direct and link transport +- **[`msg_envelope_v1`](src/NATSBridge.jl)**: Struct for the unified JSON envelope +- **[`msg_payload_v1`](src/NATSBridge.jl)**: Struct for individual payload representation +- **[`smartsend()`](src/NATSBridge.jl)**: Handles transport selection based on payload size +- **[`smartreceive()`](src/NATSBridge.jl)**: Handles both direct and link transport ### JavaScript Module: [`src/NATSBridge.js`](../src/NATSBridge.js) @@ -277,16 +278,16 @@ smartsend( ) # Even single payload must be wrapped in a list with type -smartsend("/test", [("single_data", mydata, "dictionary")]) +smartsend("/test", [("single_data", mydata, "dictionary")], nats_url="nats://localhost:4222") ``` #### Python/Micropython (Receiver) ```python from nats_bridge import smartreceive -# Receive returns a list of (dataname, data, type) tuples -payloads = smartreceive(msg) -# payloads = [(dataname1, data1, "dictionary"), (dataname2, data2, "table"), ...] +# Receive returns a dictionary with envelope metadata and payloads field +env = smartreceive(msg) +# env["payloads"] = [(dataname1, data1, "dictionary"), (dataname2, data2, "table"), ...] ``` #### JavaScript (Sender) @@ -340,8 +341,8 @@ for await (const msg of sub) { } // Also access envelope metadata - console.log(`Correlation ID: ${env.correlationId}`); - console.log(`Message ID: ${env.msgId}`); + console.log(`Correlation ID: ${env.correlation_id}`); + console.log(`Message ID: ${env.msg_id}`); } ``` @@ -359,9 +360,9 @@ df = DataFrame( category = rand(["A", "B", "C"], 10_000_000) ) -# Send via SmartSend - wrapped in a list (type is part of each tuple) -env, env_json_str = SmartSend("analysis_results", [("table_data", df, "table")]) -# env: msgEnvelope_v1 object with all metadata and payloads +# Send via smartsend - wrapped in a list (type is part of each tuple) +env, env_json_str = smartsend("analysis_results", [("table_data", df, "table")], broker_url="nats://localhost:4222") +# env: msg_envelope_v1 object with all metadata and payloads # env_json_str: JSON string representation of the envelope for publishing ``` @@ -461,7 +462,7 @@ using NATSBridge function publish_health_status(nats_url) # Send status wrapped in a list (type is part of each tuple) status = Dict("cpu" => rand(), "memory" => rand()) - smartsend("health", [("status", status, "dictionary")], nats_url=nats_url) + smartsend("health", [("status", status, "dictionary")], broker_url=nats_url) sleep(5) # Every 5 seconds end ``` @@ -523,7 +524,7 @@ def handle_device_config(msg): "device/response", [("config", config, "dictionary")], nats_url="nats://localhost:4222", - reply_to=env.get("replyTo") + reply_to=env.get("reply_to") ) ``` @@ -636,7 +637,7 @@ chat_message = [ smartsend( "chat.room123", chat_message, - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", msg_purpose="chat", reply_to="chat.room123.responses" ) @@ -684,7 +685,7 @@ await smartsend("chat.room123", message); **Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components. -**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msgEnvelope_v1` supports `AbstractArray{msgPayload_v1}` for multiple payloads. +**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads. ## Configuration @@ -700,19 +701,19 @@ await smartsend("chat.room123", message); ```json { - "correlationId": "uuid-v4-string", - "msgId": "uuid-v4-string", + "correlation_id": "uuid-v4-string", + "msg_id": "uuid-v4-string", "timestamp": "2024-01-15T10:30:00Z", - "sendTo": "topic/subject", - "msgPurpose": "ACK | NACK | updateStatus | shutdown | chat", - "senderName": "agent-wine-web-frontend", - "senderId": "uuid4", - "receiverName": "agent-backend", - "receiverId": "uuid4", - "replyTo": "topic", - "replyToMsgId": "uuid4", - "BrokerURL": "nats://localhost:4222", + "send_to": "topic/subject", + "msg_purpose": "ACK | NACK | updateStatus | shutdown | chat", + "sender_name": "agent-wine-web-frontend", + "sender_id": "uuid4", + "receiver_name": "agent-backend", + "receiver_id": "uuid4", + "reply_to": "topic", + "reply_to_msg_id": "uuid4", + "broker_url": "nats://localhost:4222", "metadata": { "content_type": "application/octet-stream", @@ -723,7 +724,7 @@ await smartsend("chat.room123", message); { "id": "uuid4", "dataname": "login_image", - "type": "image", + "payload_type": "image", "transport": "direct", "encoding": "base64", "size": 15433, diff --git a/examples/tutorial.md b/examples/tutorial.md index 220df84..9560f8a 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -182,7 +182,7 @@ for (const payload of env.payloads) { using NATSBridge # Receive and process message -env = smartreceive(msg, fileserverDownloadHandler) +env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff) for (dataname, data, type) in env["payloads"] println("Received $dataname: $data") end diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 4edc503..01bed02 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -120,15 +120,15 @@ function msg_payload_v1( metadata::Dict{String, T} = Dict{String, Any}() ) where {T<:Any} return msg_payload_v1( - id, - dataname, - payload_type, - transport, - encoding, - size, - data, - metadata - ) + id, + dataname, + payload_type, + transport, + encoding, + size, + data, + metadata + ) end @@ -167,13 +167,13 @@ payload2 = msg_payload_v1("http://example.com/file.zip", "binary"; dataname="fil # Create message envelope env = msg_envelope_v1( - "my.subject", - [payload1, payload2]; - correlation_id = string(uuid4()), - msg_purpose = "chat", - sender_name = "my-app", - receiver_name = "receiver-app", - reply_to = "reply.subject" + "my.subject", + [payload1, payload2]; + correlation_id = string(uuid4()), + msg_purpose = "chat", + sender_name = "my-app", + receiver_name = "receiver-app", + reply_to = "reply.subject" ) ``` """ @@ -498,22 +498,22 @@ function smartsend( end end - # Create msg_envelope_v1 with all payloads - env = msg_envelope_v1( - subject, - payloads; - correlation_id = cid, - msg_id = msg_id, - msg_purpose = msg_purpose, - sender_name = sender_name, - sender_id = string(uuid4()), - receiver_name = receiver_name, - receiver_id = receiver_id, - reply_to = reply_to, - reply_to_msg_id = reply_to_msg_id, - broker_url = broker_url, - metadata = Dict{String, Any}(), - ) + # Create msg_envelope_v1 with all payloads + env = msg_envelope_v1( + subject, + payloads; + correlation_id = cid, + msg_id = msg_id, + msg_purpose = msg_purpose, + sender_name = sender_name, + sender_id = string(uuid4()), + receiver_name = receiver_name, + receiver_id = receiver_id, + reply_to = reply_to, + reply_to_msg_id = reply_to_msg_id, + broker_url = broker_url, + metadata = Dict{String, Any}(), + ) env_json_str = envelope_to_json(env) # Convert envelope to JSON if is_publish @@ -602,48 +602,48 @@ function _serialize_data(data::Any, payload_type::String) """ if payload_type == "text" # Text data - convert to UTF-8 bytes - if isa(data, String) - data_bytes = Vector{UInt8}(data) # Convert string to UTF-8 bytes - return data_bytes - else - error("Text data must be a String") - end + if isa(data, String) + data_bytes = Vector{UInt8}(data) # Convert string to UTF-8 bytes + return data_bytes + else + error("Text data must be a String") + end elseif payload_type == "dictionary" # JSON data - serialize directly - json_str = JSON.json(data) # Convert Julia data to JSON string - json_str_bytes = Vector{UInt8}(json_str) # Convert JSON string to bytes - return json_str_bytes + json_str = JSON.json(data) # Convert Julia data to JSON string + json_str_bytes = Vector{UInt8}(json_str) # Convert JSON string to bytes + return json_str_bytes elseif payload_type == "table" # Table data - convert to Arrow IPC stream - io = IOBuffer() # Create in-memory buffer - Arrow.write(io, data) # Write data as Arrow IPC stream to buffer - return take!(io) # Return the buffer contents as bytes + io = IOBuffer() # Create in-memory buffer + Arrow.write(io, data) # Write data as Arrow IPC stream to buffer + return take!(io) # Return the buffer contents as bytes elseif payload_type == "image" # Image data - treat as binary - if isa(data, Vector{UInt8}) - return data # Return binary data directly - else - error("Image data must be Vector{UInt8}") - end + if isa(data, Vector{UInt8}) + return data # Return binary data directly + else + error("Image data must be Vector{UInt8}") + end elseif payload_type == "audio" # Audio data - treat as binary - if isa(data, Vector{UInt8}) - return data # Return binary data directly - else - error("Audio data must be Vector{UInt8}") - end + if isa(data, Vector{UInt8}) + return data # Return binary data directly + else + error("Audio data must be Vector{UInt8}") + end elseif payload_type == "video" # Video data - treat as binary - if isa(data, Vector{UInt8}) - return data # Return binary data directly - else - error("Video data must be Vector{UInt8}") - end + if isa(data, Vector{UInt8}) + return data # Return binary data directly + else + error("Video data must be Vector{UInt8}") + end elseif payload_type == "binary" # Binary data - treat as binary - if isa(data, IOBuffer) # Check if data is an IOBuffer - return take!(data) # Return buffer contents as bytes - elseif isa(data, Vector{UInt8}) # Check if data is already binary - return data # Return binary data directly - else # Unsupported binary data type - error("Binary data must be binary (Vector{UInt8} or IOBuffer)") - end + if isa(data, IOBuffer) # Check if data is an IOBuffer + return take!(data) # Return buffer contents as bytes + elseif isa(data, Vector{UInt8}) # Check if data is already binary + return data # Return binary data directly + else # Unsupported binary data type + error("Binary data must be binary (Vector{UInt8} or IOBuffer)") + end else # Unknown type - error("Unknown payload_type: $payload_type") + error("Unknown payload_type: $payload_type") end end @@ -673,13 +673,13 @@ connection management and logging. ``` """ function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) - conn = NATS.connect(broker_url) # Create NATS connection - try - NATS.publish(conn, subject, message) # Publish message to NATS - log_trace(correlation_id, "Message published to $subject") # Log successful publish - finally - NATS.drain(conn) # Ensure connection is closed properly - end + conn = NATS.connect(broker_url) # Create NATS connection + try + NATS.publish(conn, subject, message) # Publish message to NATS + log_trace(correlation_id, "Message published to $subject") # Log successful publish + finally + NATS.drain(conn) # Ensure connection is closed properly + end end @@ -717,60 +717,60 @@ payloads = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, ma ``` """ function smartreceive( - msg::NATS.Msg; - fileserver_download_handler::Function=_fetch_with_backoff, - max_retries::Int = 5, - base_delay::Int = 100, - max_delay::Int = 5000 + msg::NATS.Msg; + fileserver_download_handler::Function = _fetch_with_backoff, + max_retries::Int = 5, + base_delay::Int = 100, + max_delay::Int = 5000 ) - # Parse the JSON envelope - json_data = JSON.parse(String(msg.payload)) - log_trace(json_data["correlation_id"], "Processing received message") # Log message processing start + # Parse the JSON envelope + json_data = JSON.parse(String(msg.payload)) + log_trace(json_data["correlation_id"], "Processing received message") # Log message processing start + + # Process all payloads in the envelope + payloads_list = Tuple{String, Any, String}[] + + # Get number of payloads + num_payloads = length(json_data["payloads"]) - # Process all payloads in the envelope - payloads_list = Tuple{String, Any, String}[] + for i in 1:num_payloads + payload = json_data["payloads"][i] + transport = String(payload["transport"]) + dataname = String(payload["dataname"]) - # Get number of payloads - num_payloads = length(json_data["payloads"]) - - for i in 1:num_payloads - payload = json_data["payloads"][i] - transport = String(payload["transport"]) - dataname = String(payload["dataname"]) - - if transport == "direct" # Direct transport - payload is in the message - log_trace(json_data["correlation_id"], "Direct transport - decoding payload '$dataname'") # Log direct transport handling - - # Extract base64 payload from the payload - payload_b64 = String(payload["data"]) - - # Decode Base64 payload - payload_bytes = Base64.base64decode(payload_b64) # Decode base64 payload to bytes - - # Deserialize based on type - data_type = String(payload["payload_type"]) - data = _deserialize_data(payload_bytes, data_type, json_data["correlation_id"]) - - push!(payloads_list, (dataname, data, data_type)) - elseif transport == "link" # Link transport - payload is at URL - # Extract download URL from the payload - url = String(payload["data"]) - log_trace(json_data["correlation_id"], "Link transport - fetching '$dataname' from URL: $url") # Log link transport handling - - # Fetch with exponential backoff using the download handler - downloaded_data = fileserver_download_handler(url, max_retries, base_delay, max_delay, json_data["correlation_id"]) + if transport == "direct" # Direct transport - payload is in the message + log_trace(json_data["correlation_id"], "Direct transport - decoding payload '$dataname'") # Log direct transport handling + + # Extract base64 payload from the payload + payload_b64 = String(payload["data"]) + + # Decode Base64 payload + payload_bytes = Base64.base64decode(payload_b64) # Decode base64 payload to bytes + + # Deserialize based on type + data_type = String(payload["payload_type"]) + data = _deserialize_data(payload_bytes, data_type, json_data["correlation_id"]) + + push!(payloads_list, (dataname, data, data_type)) + elseif transport == "link" # Link transport - payload is at URL + # Extract download URL from the payload + url = String(payload["data"]) + log_trace(json_data["correlation_id"], "Link transport - fetching '$dataname' from URL: $url") # Log link transport handling + + # Fetch with exponential backoff using the download handler + downloaded_data = fileserver_download_handler(url, max_retries, base_delay, max_delay, json_data["correlation_id"]) - # Deserialize based on type - data_type = String(payload["payload_type"]) - data = _deserialize_data(downloaded_data, data_type, json_data["correlation_id"]) - - push!(payloads_list, (dataname, data, data_type)) - else # Unknown transport type - error("Unknown transport type for payload '$dataname': $(transport)") # Throw error for unknown transport - end + # Deserialize based on type + data_type = String(payload["payload_type"]) + data = _deserialize_data(downloaded_data, data_type, json_data["correlation_id"]) + + push!(payloads_list, (dataname, data, data_type)) + else # Unknown transport type + error("Unknown transport type for payload '$dataname': $(transport)") # Throw error for unknown transport end - json_data["payloads"] = payloads_list - return json_data # Return envelope with list of (dataname, data, data_type) tuples in payloads field + end + json_data["payloads"] = payloads_list + return json_data # Return envelope with list of (dataname, data, data_type) tuples in payloads field end @@ -805,33 +805,33 @@ data = _fetch_with_backoff("http://example.com/file.zip", 5, 100, 5000, "correla ``` """ function _fetch_with_backoff( - url::String, - max_retries::Int, - base_delay::Int, - max_delay::Int, - correlation_id::String + url::String, + max_retries::Int, + base_delay::Int, + max_delay::Int, + correlation_id::String ) - delay = base_delay # Initialize delay with base delay value - for attempt in 1:max_retries # Attempt to fetch data up to max_retries times - try - response = HTTP.request("GET", url) # Make HTTP GET request to URL - if response.status == 200 # Check if request was successful - log_trace(correlation_id, "Successfully fetched data from $url on attempt $attempt") # Log success - return response.body # Return response body as bytes - else # Request failed - error("Failed to fetch: $(response.status)") # Throw error for non-200 status - end - catch e # Handle exceptions during fetch - log_trace(correlation_id, "Attempt $attempt failed: $(typeof(e))") # Log failure - - if attempt < max_retries # Only sleep if not the last attempt - sleep(delay / 1000.0) # Sleep for delay seconds (convert from ms) - delay = min(delay * 2, max_delay) # Double delay for next attempt, capped at max_delay - end - end + delay = base_delay # Initialize delay with base delay value + for attempt in 1:max_retries # Attempt to fetch data up to max_retries times + try + response = HTTP.request("GET", url) # Make HTTP GET request to URL + if response.status == 200 # Check if request was successful + log_trace(correlation_id, "Successfully fetched data from $url on attempt $attempt") # Log success + return response.body # Return response body as bytes + else # Request failed + error("Failed to fetch: $(response.status)") # Throw error for non-200 status + end + catch e # Handle exceptions during fetch + log_trace(correlation_id, "Attempt $attempt failed: $(typeof(e))") # Log failure + + if attempt < max_retries # Only sleep if not the last attempt + sleep(delay / 1000.0) # Sleep for delay seconds (convert from ms) + delay = min(delay * 2, max_delay) # Double delay for next attempt, capped at max_delay + end end - - error("Failed to fetch data after $max_retries attempts") # Throw error if all attempts failed + end + + error("Failed to fetch data after $max_retries attempts") # Throw error if all attempts failed end @@ -875,30 +875,30 @@ table_data = _deserialize_data(arrow_bytes, "table", "correlation123") ``` """ function _deserialize_data( - data::Vector{UInt8}, - payload_type::String, - correlation_id::String + data::Vector{UInt8}, + payload_type::String, + correlation_id::String ) - if payload_type == "text" # Text data - convert to string - return String(data) # Convert bytes to string - elseif payload_type == "dictionary" # JSON data - deserialize - json_str = String(data) # Convert bytes to string - return JSON.parse(json_str) # Parse JSON string to JSON object - elseif payload_type == "table" # Table data - deserialize Arrow IPC stream - io = IOBuffer(data) # Create buffer from bytes - df = Arrow.Table(io) # Read Arrow IPC format from buffer - return df # Return DataFrame - elseif payload_type == "image" # Image data - return binary - return data # Return bytes directly - elseif payload_type == "audio" # Audio data - return binary - return data # Return bytes directly - elseif payload_type == "video" # Video data - return binary - return data # Return bytes directly - elseif payload_type == "binary" # Binary data - return binary - return data # Return bytes directly - else # Unknown type - error("Unknown payload_type: $payload_type") # Throw error for unknown type - end + if payload_type == "text" # Text data - convert to string + return String(data) # Convert bytes to string + elseif payload_type == "dictionary" # JSON data - deserialize + json_str = String(data) # Convert bytes to string + return JSON.parse(json_str) # Parse JSON string to JSON object + elseif payload_type == "table" # Table data - deserialize Arrow IPC stream + io = IOBuffer(data) # Create buffer from bytes + df = Arrow.Table(io) # Read Arrow IPC format from buffer + return df # Return DataFrame + elseif payload_type == "image" # Image data - return binary + return data # Return bytes directly + elseif payload_type == "audio" # Audio data - return binary + return data # Return bytes directly + elseif payload_type == "video" # Video data - return binary + return data # Return bytes directly + elseif payload_type == "binary" # Binary data - return binary + return data # Return bytes directly + else # Unknown type + error("Unknown payload_type: $payload_type") # Throw error for unknown type + end end @@ -1035,10 +1035,10 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) url_upload = "$file_server_url/file/$uploadid" headers = ["X-UploadToken" => uploadtoken] http_response = open(filepath, "r") do file_stream - form = HTTP.Form(Dict("file" => file_stream)) - - # Adding status_exception=false prevents 4xx/5xx from triggering 'catch' - HTTP.post(url_upload, headers, form; status_exception = false) + form = HTTP.Form(Dict("file" => file_stream)) + + # Adding status_exception=false prevents 4xx/5xx from triggering 'catch' + HTTP.post(url_upload, headers, form; status_exception = false) end if !isnothing(http_response) && http_response.status == 200 @@ -1047,7 +1047,6 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) error("Failed to upload file: server returned status $(http_response.status)") end response_json = JSON.parse(http_response.body) - fileid = response_json["id"] # url of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" diff --git a/test/test_julia_dict_sender.jl b/test/test_julia_dict_sender.jl index c180320..dd2a847 100644 --- a/test/test_julia_dict_sender.jl +++ b/test/test_julia_dict_sender.jl @@ -95,9 +95,9 @@ function test_dict_send() env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples - nats_url = NATS_URL, + broker_url = NATS_URL, fileserver_url = FILESERVER_URL, - fileserverUploadHandler = plik_upload_handler, + fileserver_upload_handler = plik_upload_handler, size_threshold = 1_000_000, # 1MB threshold correlation_id = correlation_id, msg_purpose = "chat", @@ -115,7 +115,7 @@ function test_dict_send() for (i, payload) in enumerate(env.payloads) log_trace("Payload $i ('$payload.dataname'):") log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.type)") + log_trace(" Type: $(payload.payload_type)") log_trace(" Size: $(payload.size) bytes") log_trace(" Encoding: $(payload.encoding)") diff --git a/test/test_julia_file_sender.jl b/test/test_julia_file_sender.jl index db1bed1..abce1c2 100644 --- a/test/test_julia_file_sender.jl +++ b/test/test_julia_file_sender.jl @@ -82,9 +82,9 @@ function test_large_binary_send() env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples - nats_url = NATS_URL; + broker_url = NATS_URL; fileserver_url = FILESERVER_URL, - fileserverUploadHandler = plik_upload_handler, + fileserver_upload_handler = plik_upload_handler, size_threshold = 1_000_000, correlation_id = correlation_id, msg_purpose = "chat", @@ -97,7 +97,7 @@ function test_large_binary_send() ) log_trace("Sent message with transport: $(env.payloads[1].transport)") - log_trace("Envelope type: $(env.payloads[1].type)") + log_trace("Envelope type: $(env.payloads[1].payload_type)") # Check if link transport was used if env.payloads[1].transport == "link" diff --git a/test/test_julia_mix_payloads_sender.jl b/test/test_julia_mix_payloads_sender.jl index f62e009..541a4d0 100644 --- a/test/test_julia_mix_payloads_sender.jl +++ b/test/test_julia_mix_payloads_sender.jl @@ -189,9 +189,9 @@ function test_mix_send() env, env_json_str = NATSBridge.smartsend( SUBJECT, payloads; # List of (dataname, data, type) tuples - nats_url = NATS_URL, + broker_url = NATS_URL, fileserver_url = FILESERVER_URL, - fileserverUploadHandler = plik_upload_handler, + fileserver_upload_handler = plik_upload_handler, size_threshold = 1_000_000, # 1MB threshold correlation_id = correlation_id, msg_purpose = "chat", @@ -209,7 +209,7 @@ function test_mix_send() for (i, payload) in enumerate(env.payloads) log_trace("Payload $i ('$payload.dataname'):") log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.type)") + log_trace(" Type: $(payload.payload_type)") log_trace(" Size: $(payload.size) bytes") log_trace(" Encoding: $(payload.encoding)") diff --git a/test/test_julia_table_sender.jl b/test/test_julia_table_sender.jl index 75e93f3..386ad50 100644 --- a/test/test_julia_table_sender.jl +++ b/test/test_julia_table_sender.jl @@ -93,9 +93,9 @@ function test_table_send() env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples - nats_url = NATS_URL, + broker_url = NATS_URL, fileserver_url = FILESERVER_URL, - fileserverUploadHandler = plik_upload_handler, + fileserver_upload_handler = plik_upload_handler, size_threshold = 1_000_000, # 1MB threshold correlation_id = correlation_id, msg_purpose = "chat", @@ -113,7 +113,7 @@ function test_table_send() for (i, payload) in enumerate(env.payloads) log_trace("Payload $i ('$payload.dataname'):") log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.type)") + log_trace(" Type: $(payload.payload_type)") log_trace(" Size: $(payload.size) bytes") log_trace(" Encoding: $(payload.encoding)") diff --git a/test/test_julia_text_sender.jl b/test/test_julia_text_sender.jl index 6625d16..29e1839 100644 --- a/test/test_julia_text_sender.jl +++ b/test/test_julia_text_sender.jl @@ -78,9 +78,9 @@ function test_text_send() env, env_json_str = NATSBridge.smartsend( SUBJECT, [data1, data2]; # List of (dataname, data, type) tuples - nats_url = NATS_URL, + broker_url = NATS_URL, fileserver_url = FILESERVER_URL, - fileserverUploadHandler = plik_upload_handler, + fileserver_upload_handler = plik_upload_handler, size_threshold = 1_000_000, # 1MB threshold correlation_id = correlation_id, msg_purpose = "chat", @@ -98,7 +98,7 @@ function test_text_send() for (i, payload) in enumerate(env.payloads) log_trace("Payload $i ('$payload.dataname'):") log_trace(" Transport: $(payload.transport)") - log_trace(" Type: $(payload.type)") + log_trace(" Type: $(payload.payload_type)") log_trace(" Size: $(payload.size) bytes") log_trace(" Encoding: $(payload.encoding)") -- 2.49.1 From 5a9e93d6e77f344cbe005921326f16d0b5f87462 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 20:38:45 +0700 Subject: [PATCH 15/35] update --- docs/architecture.md | 165 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 140 insertions(+), 25 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 31e5c99..215bfbf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -464,60 +464,175 @@ end #### Dependencies - `nats.js` - Core NATS functionality - `apache-arrow` - Arrow IPC serialization -- `uuid` - Correlation ID generation +- `uuid` - Correlation ID and message ID generation +- `base64-arraybuffer` - Base64 encoding/decoding +- `node-fetch` or `fetch` - HTTP client for file server #### smartsend Function ```javascript -async function smartsend(subject, data, options = {}) - // data format: [(dataname, data, type), ...] - // options object should include: - // - natsUrl: NATS server URL - // - fileserverUrl: base URL of the file server - // - sizeThreshold: threshold in bytes for transport selection - // - correlationId: optional correlation ID for tracing +async function smartsend( + subject, + data, // List of (dataname, data, type) tuples: [(dataname1, data1, type1), ...] + options = {} +) ``` +**Options:** +- `broker_url` (String) - NATS server URL (default: `"nats://localhost:4222"`) +- `fileserver_url` (String) - Base URL of the file server (default: `"http://localhost:8080"`) +- `size_threshold` (Number) - Threshold in bytes for transport selection (default: `1048576` = 1MB) +- `correlation_id` (String) - Optional correlation ID for tracing +- `msg_purpose` (String) - Purpose of the message (default: `"chat"`) +- `sender_name` (String) - Sender name (default: `"NATSBridge"`) +- `receiver_name` (String) - Message receiver name (default: `""`) +- `receiver_id` (String) - Message receiver ID (default: `""`) +- `reply_to` (String) - Topic to reply to (default: `""`) +- `reply_to_msg_id` (String) - Message ID this message is replying to (default: `""`) +- `fileserver_upload_handler` (Function) - Custom upload handler function + +**Return Value:** +- Returns a Promise that resolves to an object containing: + - `envelope` - The envelope object containing all metadata and payloads + - `envelope_json` - JSON string representation of the envelope for publishing + - `published` - Boolean indicating whether the message was automatically published to NATS + **Input Format:** - `data` - **Must be a list of (dataname, data, type) tuples**: `[(dataname1, data1, "type1"), (dataname2, data2, "type2"), ...]` - Even for single payloads: `[(dataname1, data1, "type1")]` - Each payload can have a different type, enabling mixed-content messages +- Supported types: `"text"`, `"dictionary"`, `"table"`, `"image"`, `"audio"`, `"video"`, `"binary"` **Flow:** -1. Iterate through the list of (dataname, data, type) tuples -2. For each payload: extract the type from the tuple and serialize accordingly -3. Check payload size -4. If < threshold: publish directly to NATS -5. If >= threshold: upload to HTTP server, publish NATS with URL +1. Generate correlation ID and message ID if not provided +2. Iterate through the list of `(dataname, data, type)` tuples +3. For each payload: + - Serialize based on payload type + - Check payload size + - If < threshold: Base64 encode and include in envelope + - If >= threshold: Upload to HTTP server, store URL in envelope +4. Publish the JSON envelope to NATS +5. Return envelope object and JSON string #### smartreceive Handler ```javascript async function smartreceive(msg, options = {}) - // options object should include: - // - fileserverDownloadHandler: function to fetch data from file server URL - // - max_retries: maximum retry attempts for fetching URL - // - base_delay: initial delay for exponential backoff in ms - // - max_delay: maximum delay for exponential backoff in ms - // - correlationId: optional correlation ID for tracing ``` +**Options:** +- `fileserver_download_handler` (Function) - Custom download handler function +- `max_retries` (Number) - Maximum retry attempts for fetching URL (default: `5`) +- `base_delay` (Number) - Initial delay for exponential backoff in ms (default: `100`) +- `max_delay` (Number) - Maximum delay for exponential backoff in ms (default: `5000`) +- `correlation_id` (String) - Optional correlation ID for tracing + **Output Format:** -- Returns a dictionary (key-value map) containing all envelope fields: - - `correlationId`, `msgId`, `timestamp`, `sendTo`, `msgPurpose`, `senderName`, `senderId`, `receiverName`, `receiverId`, `replyTo`, `replyToMsgId`, `brokerURL` +- Returns a Promise that resolves to an object containing all envelope fields: + - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - `metadata` - Message-level metadata dictionary - - `payloads` - List of dictionaries, each containing deserialized payload data + - `payloads` - List of dictionaries, each containing deserialized payload data with keys: `dataname`, `data`, `payload_type` **Process Flow:** 1. Parse the JSON envelope to extract all fields -2. Iterate through each payload in `payloads` +2. Iterate through each payload in `payloads` array 3. For each payload: - Determine transport type (`direct` or `link`) - - If `direct`: decode Base64 data from the message - - If `link`: fetch data from URL using exponential backoff + - If `direct`: Base64 decode the data from the message + - If `link`: Fetch data from URL using exponential backoff (via `fileserverDownloadHandler`) + - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) +4. Return envelope object with `payloads` field containing list of `(dataname, data, type)` tuples + +**Note:** The `fileserver_download_handler` receives `(url, max_retries, base_delay, max_delay, correlation_id)` and returns `ArrayBuffer` or `Uint8Array`. + +### Python/Micropython Implementation + +#### Dependencies +- `nats-python` - Core NATS functionality +- `pyarrow` - Arrow IPC serialization +- `uuid` - Correlation ID and message ID generation +- `base64` - Base64 encoding/decoding +- `requests` or `aiohttp` - HTTP client for file server + +#### smartsend Function + +```python +async def smartsend( + subject: str, + data: List[Tuple[str, Any, str]], # List of (dataname, data, type) tuples + options: Dict = {} +) +``` + +**Options:** +- `broker_url` (str) - NATS server URL (default: `"nats://localhost:4222"`) +- `fileserver_url` (str) - Base URL of the file server (default: `"http://localhost:8080"`) +- `size_threshold` (int) - Threshold in bytes for transport selection (default: `1048576` = 1MB) +- `correlation_id` (str) - Optional correlation ID for tracing +- `msg_purpose` (str) - Purpose of the message (default: `"chat"`) +- `sender_name` (str) - Sender name (default: `"NATSBridge"`) +- `receiver_name` (str) - Message receiver name (default: `""`) +- `receiver_id` (str) - Message receiver ID (default: `""`) +- `reply_to` (str) - Topic to reply to (default: `""`) +- `reply_to_msg_id` (str) - Message ID this message is replying to (default: `""`) +- `fileserver_upload_handler` (Callable) - Custom upload handler function + +**Return Value:** +- Returns a tuple `(envelope, envelope_json)` where: + - `envelope` - The envelope dictionary containing all metadata and payloads + - `envelope_json` - JSON string representation of the envelope for publishing + +**Input Format:** +- `data` - **Must be a list of (dataname, data, type) tuples**: `[(dataname1, data1, "type1"), (dataname2, data2, "type2"), ...]` +- Even for single payloads: `[(dataname1, data1, "type1")]` +- Each payload can have a different type, enabling mixed-content messages +- Supported types: `"text"`, `"dictionary"`, `"table"`, `"image"`, `"audio"`, `"video"`, `"binary"` + +**Flow:** +1. Generate correlation ID and message ID if not provided +2. Iterate through the list of `(dataname, data, type)` tuples +3. For each payload: + - Serialize based on payload type + - Check payload size + - If < threshold: Base64 encode and include in envelope + - If >= threshold: Upload to HTTP server, store URL in envelope +4. Publish the JSON envelope to NATS +5. Return envelope dictionary and JSON string + +#### smartreceive Handler + +```python +async def smartreceive( + msg: NATS.Message, + options: Dict = {} +) +``` + +**Options:** +- `fileserver_download_handler` (Callable) - Custom download handler function +- `max_retries` (int) - Maximum retry attempts for fetching URL (default: `5`) +- `base_delay` (int) - Initial delay for exponential backoff in ms (default: `100`) +- `max_delay` (int) - Maximum delay for exponential backoff in ms (default: `5000`) +- `correlation_id` (str) - Optional correlation ID for tracing + +**Output Format:** +- Returns a dictionary containing all envelope fields: + - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` + - `metadata` - Message-level metadata dictionary + - `payloads` - List of tuples, each containing `(dataname, data, payload_type)` with deserialized payload data + +**Process Flow:** +1. Parse the JSON envelope to extract all fields +2. Iterate through each payload in `payloads` list +3. For each payload: + - Determine transport type (`direct` or `link`) + - If `direct`: Base64 decode the data from the message + - If `link`: Fetch data from URL using exponential backoff (via `fileserver_download_handler`) - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) 4. Return envelope dictionary with `payloads` field containing list of `(dataname, data, type)` tuples +**Note:** The `fileserver_download_handler` receives `(url: str, max_retries: int, base_delay: int, max_delay: int, correlation_id: str)` and returns `bytes`. + ## Scenario Implementations ### Scenario 1: Command & Control (Small Dictionary) -- 2.49.1 From ab20cd896f1c76230b4fdff97cd9ae70afafb1cb Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 21:18:19 +0700 Subject: [PATCH 16/35] update --- docs/architecture.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 215bfbf..81ac9cd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -493,8 +493,8 @@ async function smartsend( **Return Value:** - Returns a Promise that resolves to an object containing: - - `envelope` - The envelope object containing all metadata and payloads - - `envelope_json` - JSON string representation of the envelope for publishing + - `env` - The envelope object containing all metadata and payloads + - `env_json_str` - JSON string representation of the envelope for publishing - `published` - Boolean indicating whether the message was automatically published to NATS **Input Format:** @@ -539,7 +539,7 @@ async function smartreceive(msg, options = {}) 3. For each payload: - Determine transport type (`direct` or `link`) - If `direct`: Base64 decode the data from the message - - If `link`: Fetch data from URL using exponential backoff (via `fileserverDownloadHandler`) + - If `link`: Fetch data from URL using exponential backoff (via `fileserver_download_handler`) - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) 4. Return envelope object with `payloads` field containing list of `(dataname, data, type)` tuples @@ -578,9 +578,9 @@ async def smartsend( - `fileserver_upload_handler` (Callable) - Custom upload handler function **Return Value:** -- Returns a tuple `(envelope, envelope_json)` where: - - `envelope` - The envelope dictionary containing all metadata and payloads - - `envelope_json` - JSON string representation of the envelope for publishing +- Returns a tuple `(env, env_json_str)` where: + - `env` - The envelope dictionary containing all metadata and payloads + - `env_json_str` - JSON string representation of the envelope for publishing **Input Format:** - `data` - **Must be a list of (dataname, data, type) tuples**: `[(dataname1, data1, "type1"), (dataname2, data2, "type2"), ...]` -- 2.49.1 From 7f68d08134d783f2b9fb2693f6703c1c2ecac383 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 24 Feb 2026 21:40:33 +0700 Subject: [PATCH 17/35] update --- docs/implementation.md | 121 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/docs/implementation.md b/docs/implementation.md index 7b68f11..1f4b6e4 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -19,17 +19,39 @@ NATSBridge is implemented in three languages, each providing the same API: | **JavaScript** | [`src/NATSBridge.js`](../src/NATSBridge.js) | JavaScript implementation for Node.js and browsers | | **Python/Micropython** | [`src/nats_bridge.py`](../src/nats_bridge.py) | Python implementation for desktop and microcontrollers | -### Multi-Payload Support +### File Server Handler Architecture -The implementation uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.** +The system uses **handler functions** to abstract file server operations, allowing support for different file server implementations (e.g., Plik, AWS S3, custom HTTP server). + +**Handler Function Signatures:** + +```julia +# Upload handler - uploads data to file server and returns URL +# The handler is passed to smartsend as fileserver_upload_handler parameter +# It receives: (file_server_url::String, dataname::String, data::Vector{UInt8}) +# Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url" +fileserver_upload_handler(file_server_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} + +# Download handler - fetches data from file server URL with exponential backoff +# The handler is passed to smartreceive as fileserver_download_handler parameter +# It receives: (url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String) +# Returns: Vector{UInt8} (the downloaded data) +fileserver_download_handler(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} +``` + +This design allows the system to support multiple file server backends without changing the core messaging logic. + +### Multi-Payload Support (Standard API) + +The system uses a **standardized list-of-tuples format** for all payload operations. **Even when sending a single payload, the user must wrap it in a list.** **API Standard:** ```julia # Input format for smartsend (always a list of tuples with type info) [(dataname1, data1, type1), (dataname2, data2, type2), ...] -# Output format for smartreceive (returns envelope dictionary with payloads field) -# Returns: Dict with envelope metadata and payloads field containing list of tuples +# Output format for smartreceive (returns a dictionary with payloads field containing list of tuples) +# Returns: Dict with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}} # { # "correlation_id": "...", # "msg_id": "...", @@ -48,20 +70,51 @@ The implementation uses a **standardized list-of-tuples format** for all payload # } ``` -Where `type` can be: `"text"`, `"dictionary"`, `"table"`, `"image"`, `"audio"`, `"video"`, `"binary"` +**Supported Types:** +- `"text"` - Plain text +- `"dictionary"` - JSON-serializable dictionaries (Dict, NamedTuple) +- `"table"` - Tabular data (DataFrame, array of structs) +- `"image"` - Image data (Bitmap, PNG/JPG bytes) +- `"audio"` - Audio data (WAV, MP3 bytes) +- `"video"` - Video data (MP4, AVI bytes) +- `"binary"` - Generic binary data (Vector{UInt8}) + +This design allows per-payload type specification, enabling **mixed-content messages** where different payloads can use different serialization formats in a single message. **Examples:** ```julia -# Single payload - still wrapped in a list (type is required as third element) -smartsend("/test", [(dataname1, data1, "text")], broker_url="nats://localhost:4222") +# Single payload - still wrapped in a list +smartsend( + "/test", + [("dataname1", data1, "dictionary")], # List with one tuple (data, type) + broker_url="nats://localhost:4222", + fileserver_upload_handler=plik_oneshot_upload +) -# Multiple payloads in one message (each payload has its own type) -smartsend("/test", [(dataname1, data1, "dictionary"), (dataname2, data2, "table")], broker_url="nats://localhost:4222") +# Multiple payloads in one message with different types +smartsend( + "/test", + [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")], + broker_url="nats://localhost:4222", + fileserver_upload_handler=plik_oneshot_upload +) + +# Mixed content (e.g., chat with text, image, audio) +smartsend( + "/chat", + [ + ("message_text", "Hello!", "text"), + ("user_image", image_data, "image"), + ("audio_clip", audio_data, "audio") + ], + broker_url="nats://localhost:4222" +) # Receive returns a dictionary envelope with all metadata and deserialized payloads env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) -# env["payloads"] = [(dataname1, data1, "text"), (dataname2, data2, "table"), ...] +# env["payloads"] = [("dataname1", data1, type1), ("dataname2", data2, type2), ...] # env["correlation_id"], env["msg_id"], etc. +# env is a dictionary containing envelope metadata and payloads field ``` ## Cross-Platform Interoperability @@ -263,7 +316,53 @@ node test/scenario3_julia_to_julia.js ## Usage -### Scenario 0: Basic Multi-Payload Example +### Scenario 1: Command & Control (Small Dictionary) + +**Focus:** Sending small dictionary configurations across platforms. This is the simplest use case for command and control scenarios. + +**Julia (Sender/Receiver):** +```julia +using NATSBridge + +# Subscribe to control subject +# Parse JSON envelope +# Execute simulation with parameters +# Send acknowledgment +``` + +**JavaScript (Sender/Receiver):** +```javascript +const { smartsend } = require('./src/NATSBridge'); + +// Create small dictionary config +// Send via smartsend with type="dictionary" +const config = { + step_size: 0.01, + iterations: 1000, + threshold: 0.5 +}; + +await smartsend("control", [ + { dataname: "config", data: config, type: "dictionary" } +]); +``` + +**Python/Micropython (Sender/Receiver):** +```python +from nats_bridge import smartsend + +# Create small dictionary config +# Send via smartsend with type="dictionary" +config = { + "step_size": 0.01, + "iterations": 1000, + "threshold": 0.5 +} + +smartsend("control", [("config", config, "dictionary")]) +``` + +### Basic Multi-Payload Example #### Python/Micropython (Sender) ```python -- 2.49.1 From 1a10bc1a5f1ba0689187305587a28ecd8d61ce61 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 05:32:59 +0700 Subject: [PATCH 18/35] update --- docs/architecture.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 81ac9cd..f7ccb6a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -445,7 +445,7 @@ end - Returns a dictionary (key-value map) containing all envelope fields: - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - `metadata` - Message-level metadata dictionary - - `payloads` - List of dictionaries, each containing deserialized payload data + - `payloads` - List of tuples, each containing `(dataname, data, type)` with deserialized payload data **Process Flow:** 1. Parse the JSON envelope to extract all fields @@ -531,16 +531,16 @@ async function smartreceive(msg, options = {}) - Returns a Promise that resolves to an object containing all envelope fields: - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - `metadata` - Message-level metadata dictionary - - `payloads` - List of dictionaries, each containing deserialized payload data with keys: `dataname`, `data`, `payload_type` + - `payloads` - List of tuples, each containing `(dataname, data, type)` with deserialized payload data **Process Flow:** 1. Parse the JSON envelope to extract all fields 2. Iterate through each payload in `payloads` array 3. For each payload: - - Determine transport type (`direct` or `link`) - - If `direct`: Base64 decode the data from the message - - If `link`: Fetch data from URL using exponential backoff (via `fileserver_download_handler`) - - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) + - Determine transport type (`direct` or `link`) + - If `direct`: Base64 decode the data from the message + - If `link`: Fetch data from URL using exponential backoff (via `fileserver_download_handler`) + - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) 4. Return envelope object with `payloads` field containing list of `(dataname, data, type)` tuples **Note:** The `fileserver_download_handler` receives `(url, max_retries, base_delay, max_delay, correlation_id)` and returns `ArrayBuffer` or `Uint8Array`. @@ -824,7 +824,7 @@ async def smartreceive( **Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components across all platforms. -**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msgEnvelope_v1` supports `AbstractArray{msgPayload_v1}` for multiple payloads. +**Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads. ## Performance Considerations -- 2.49.1 From 61d81bed6268900f52b83113d4903a272d838be3 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 06:04:40 +0700 Subject: [PATCH 19/35] update --- src/NATSBridge.jl | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 01bed02..ed9ddd4 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -436,7 +436,6 @@ function smartsend( # Generate correlation ID if not provided cid = correlation_id !== nothing ? correlation_id : string(uuid4()) # Create or use provided correlation ID - log_trace(cid, "Starting smartsend for subject: $subject") # Log start of send operation # Generate message metadata @@ -706,7 +705,7 @@ A HTTP file server is required along with its download function. - `max_delay::Int = 5000` - Maximum delay for exponential backoff in ms # Return: - - `Vector{Tuple{String, Any, String}}` - List of (dataname, data, type) tuples + - JSON object of envelope with list of (dataname, data, data_type) tuples in payloads field # Example ```jldoctest @@ -724,22 +723,22 @@ function smartreceive( max_delay::Int = 5000 ) # Parse the JSON envelope - json_data = JSON.parse(String(msg.payload)) - log_trace(json_data["correlation_id"], "Processing received message") # Log message processing start + env_json_obj = JSON.parse(String(msg.payload)) + log_trace(env_json_obj["correlation_id"], "Processing received message") # Log message processing start # Process all payloads in the envelope payloads_list = Tuple{String, Any, String}[] # Get number of payloads - num_payloads = length(json_data["payloads"]) + num_payloads = length(env_json_obj["payloads"]) for i in 1:num_payloads - payload = json_data["payloads"][i] + payload = env_json_obj["payloads"][i] transport = String(payload["transport"]) dataname = String(payload["dataname"]) if transport == "direct" # Direct transport - payload is in the message - log_trace(json_data["correlation_id"], "Direct transport - decoding payload '$dataname'") # Log direct transport handling + log_trace(env_json_obj["correlation_id"], "Direct transport - decoding payload '$dataname'") # Log direct transport handling # Extract base64 payload from the payload payload_b64 = String(payload["data"]) @@ -749,28 +748,28 @@ function smartreceive( # Deserialize based on type data_type = String(payload["payload_type"]) - data = _deserialize_data(payload_bytes, data_type, json_data["correlation_id"]) + data = _deserialize_data(payload_bytes, data_type, env_json_obj["correlation_id"]) push!(payloads_list, (dataname, data, data_type)) elseif transport == "link" # Link transport - payload is at URL # Extract download URL from the payload url = String(payload["data"]) - log_trace(json_data["correlation_id"], "Link transport - fetching '$dataname' from URL: $url") # Log link transport handling + log_trace(env_json_obj["correlation_id"], "Link transport - fetching '$dataname' from URL: $url") # Log link transport handling # Fetch with exponential backoff using the download handler - downloaded_data = fileserver_download_handler(url, max_retries, base_delay, max_delay, json_data["correlation_id"]) + downloaded_data = fileserver_download_handler(url, max_retries, base_delay, max_delay, env_json_obj["correlation_id"]) # Deserialize based on type data_type = String(payload["payload_type"]) - data = _deserialize_data(downloaded_data, data_type, json_data["correlation_id"]) + data = _deserialize_data(downloaded_data, data_type, env_json_obj["correlation_id"]) push!(payloads_list, (dataname, data, data_type)) else # Unknown transport type error("Unknown transport type for payload '$dataname': $(transport)") # Throw error for unknown transport end end - json_data["payloads"] = payloads_list - return json_data # Return envelope with list of (dataname, data, data_type) tuples in payloads field + env_json_obj["payloads"] = payloads_list + return env_json_obj # JSON object of envelope with list of (dataname, data, data_type) tuples in payloads field end @@ -929,7 +928,7 @@ retrieves an upload ID and token, then uploads the file data as multipart form d ```jldoctest using HTTP, JSON - file_server_url = "http://localhost:8080" + fileserver_url = "http://localhost:8080" filename = "test.txt" data = Vector{UInt8}("hello world") @@ -1006,7 +1005,7 @@ retrieves an upload ID and token, then uploads the file data as multipart form d ```jldoctest using HTTP, JSON -file_server_url = "http://localhost:8080" +fileserver_url = "http://localhost:8080" filepath = "./test.zip" # Upload to local plik server -- 2.49.1 From 14b3790251cd1fb718f3e6b737e745640f6aa8aa Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 06:23:24 +0700 Subject: [PATCH 20/35] update --- docs/architecture.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f7ccb6a..09dac03 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,9 +18,9 @@ The system uses **handler functions** to abstract file server operations, allowi ```julia # Upload handler - uploads data to file server and returns URL # The handler is passed to smartsend as fileserver_upload_handler parameter -# It receives: (file_server_url::String, dataname::String, data::Vector{UInt8}) +# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8}) # Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url" -fileserver_upload_handler(file_server_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} +fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} # Download handler - fetches data from file server URL with exponential backoff # The handler is passed to smartreceive as fileserver_download_handler parameter @@ -40,8 +40,8 @@ The system uses a **standardized list-of-tuples format** for all payload operati # Input format for smartsend (always a list of tuples with type info) [(dataname1, data1, type1), (dataname2, data2, type2), ...] -# Output format for smartreceive (returns a dictionary with payloads field containing list of tuples) -# Returns: Dict with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}} +# Output format for smartreceive (returns a dictionary-like object with payloads field containing list of tuples) +# Returns: Dict-like object with envelope metadata and payloads field containing Vector{Tuple{String, Any, String}} # { # "correlation_id": "...", # "msg_id": "...", @@ -442,7 +442,7 @@ end ``` **Output Format:** -- Returns a dictionary (key-value map) containing all envelope fields: +- Returns a JSON object (dictionary) containing all envelope fields: - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - `metadata` - Message-level metadata dictionary - `payloads` - List of tuples, each containing `(dataname, data, type)` with deserialized payload data @@ -495,7 +495,6 @@ async function smartsend( - Returns a Promise that resolves to an object containing: - `env` - The envelope object containing all metadata and payloads - `env_json_str` - JSON string representation of the envelope for publishing - - `published` - Boolean indicating whether the message was automatically published to NATS **Input Format:** - `data` - **Must be a list of (dataname, data, type) tuples**: `[(dataname1, data1, "type1"), (dataname2, data2, "type2"), ...]` @@ -616,7 +615,7 @@ async def smartreceive( - `correlation_id` (str) - Optional correlation ID for tracing **Output Format:** -- Returns a dictionary containing all envelope fields: +- Returns a JSON object (dictionary) containing all envelope fields: - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - `metadata` - Message-level metadata dictionary - `payloads` - List of tuples, each containing `(dataname, data, payload_type)` with deserialized payload data -- 2.49.1 From 6a42ba7e43e1c3f50b5e9b5c6f4e389fbabb3c0d Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 07:29:42 +0700 Subject: [PATCH 21/35] update --- AI_prompt.txt | 4 +++- src/NATSBridge.jl | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/AI_prompt.txt b/AI_prompt.txt index 6a0f3f3..38e74a4 100644 --- a/AI_prompt.txt +++ b/AI_prompt.txt @@ -12,4 +12,6 @@ Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement -Create a walkthrough for Julia service-A service sending a mix-content chat message to Julia service-B. the chat message must includes \ No newline at end of file +Create a walkthrough for Julia service-A service sending a mix-content chat message to Julia service-B. the chat message must includes + +I update architecture.md and NATSBridge.jl. Use them as ground truth and update implementation.md accordingly. Also look for any inconsistency. \ No newline at end of file diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index ed9ddd4..03044fe 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -914,7 +914,7 @@ retrieves an upload ID and token, then uploads the file data as multipart form d # Arguments: - `file_server_url::String` - Base URL of the plik server (e.g., `"http://localhost:8080"`) - - `filename::String` - Name of the file being uploaded + - `dataname::String` - Name of the file being uploaded - `data::Vector{UInt8}` - Raw byte data of the file content # Return: @@ -929,17 +929,17 @@ retrieves an upload ID and token, then uploads the file data as multipart form d using HTTP, JSON fileserver_url = "http://localhost:8080" - filename = "test.txt" + dataname = "test.txt" data = Vector{UInt8}("hello world") # Upload to local plik server - result = plik_oneshot_upload(file_server_url, filename, data) + result = plik_oneshot_upload(file_server_url, dataname, data) # Access the result as a Dict # result["status"], result["uploadid"], result["fileid"], result["url"] ``` """ -function plik_oneshot_upload(file_server_url::String, filename::String, data::Vector{UInt8}) +function plik_oneshot_upload(file_server_url::String, dataname::String, data::Vector{UInt8}) # ----------------------------------------- get upload id ---------------------------------------- # # Equivalent curl command: curl -X POST -d '{ "OneShot" : true }' http://localhost:8080/upload @@ -953,7 +953,7 @@ function plik_oneshot_upload(file_server_url::String, filename::String, data::Ve # ------------------------------------------ upload file ----------------------------------------- # # Equivalent curl command: curl -X POST --header "X-UploadToken: UPLOAD_TOKEN" -F "file=@PATH_TO_FILE" http://localhost:8080/file/UPLOAD_ID - file_multipart = HTTP.Multipart(filename, IOBuffer(data), "application/octet-stream") # Plik won't accept raw bytes upload + file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream") # Plik won't accept raw bytes upload url_upload = "$file_server_url/file/$uploadid" headers = ["X-UploadToken" => uploadtoken] @@ -973,7 +973,7 @@ function plik_oneshot_upload(file_server_url::String, filename::String, data::Ve fileid = response_json["id"] # url of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" - url = "$file_server_url/file/$uploadid/$fileid/$filename" + url = "$file_server_url/file/$uploadid/$fileid/$dataname" return Dict("status" => http_response.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url) end -- 2.49.1 From 8c793a81b64f73c4430e9acbfb5d81669d2cfa2d Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 08:02:03 +0700 Subject: [PATCH 22/35] update --- AI_prompt.txt | 4 +- docs/implementation.md | 85 ++++++++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/AI_prompt.txt b/AI_prompt.txt index 38e74a4..b6ad565 100644 --- a/AI_prompt.txt +++ b/AI_prompt.txt @@ -14,4 +14,6 @@ Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement Create a walkthrough for Julia service-A service sending a mix-content chat message to Julia service-B. the chat message must includes -I update architecture.md and NATSBridge.jl. Use them as ground truth and update implementation.md accordingly. Also look for any inconsistency. \ No newline at end of file +I update architecture.md and NATSBridge.jl. Use them as ground truth and update implementation.md accordingly. Also look for any inconsistency. + + diff --git a/docs/implementation.md b/docs/implementation.md index 1f4b6e4..5792e3b 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -28,9 +28,9 @@ The system uses **handler functions** to abstract file server operations, allowi ```julia # Upload handler - uploads data to file server and returns URL # The handler is passed to smartsend as fileserver_upload_handler parameter -# It receives: (file_server_url::String, dataname::String, data::Vector{UInt8}) +# It receives: (fileserver_url::String, dataname::String, data::Vector{UInt8}) # Returns: Dict{String, Any} with keys: "status", "uploadid", "fileid", "url" -fileserver_upload_handler(file_server_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} +fileserver_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any} # Download handler - fetches data from file server URL with exponential backoff # The handler is passed to smartreceive as fileserver_download_handler parameter @@ -148,10 +148,12 @@ NATSBridge is designed for seamless communication between Julia, JavaScript, and ### Example: Julia ↔ Python ↔ JavaScript ```julia -# Julia sender +# Julia sender - smartsend returns (env, env_json_str) using NATSBridge data = [("message", "Hello from Julia!", "text")] -smartsend("/cross_platform", data, broker_url="nats://localhost:4222") +env, env_json_str = smartsend("/cross_platform", data, broker_url="nats://localhost:4222") +# env: msg_envelope_v1 with all metadata and payloads +# env_json_str: JSON string for publishing ``` ```javascript @@ -165,7 +167,7 @@ const env = await smartreceive(msg); # Python sender from nats_bridge import smartsend data = [("response", "Hello from Python!", "text")] -smartsend("/cross_platform", data, nats_url="nats://localhost:4222") +smartsend("/cross_platform", data, broker_url="nats://localhost:4222") ``` All three platforms can communicate seamlessly using the same NATS subjects and data format. @@ -324,10 +326,15 @@ node test/scenario3_julia_to_julia.js ```julia using NATSBridge -# Subscribe to control subject -# Parse JSON envelope -# Execute simulation with parameters -# Send acknowledgment +# Send small dictionary config (wrapped in list with type) +config = Dict("step_size" => 0.01, "iterations" => 1000, "threshold" => 0.5) +env, env_json_str = smartsend( + "control", + [("config", config, "dictionary")], + broker_url="nats://localhost:4222" +) +# env: msg_envelope_v1 with all metadata and payloads +# env_json_str: JSON string for publishing ``` **JavaScript (Sender/Receiver):** @@ -372,12 +379,12 @@ from nats_bridge import smartsend smartsend( "/test", [("dataname1", data1, "dictionary"), ("dataname2", data2, "table")], - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", fileserver_url="http://localhost:8080" ) # Even single payload must be wrapped in a list with type -smartsend("/test", [("single_data", mydata, "dictionary")], nats_url="nats://localhost:4222") +smartsend("/test", [("single_data", mydata, "dictionary")], broker_url="nats://localhost:4222") ``` #### Python/Micropython (Receiver) @@ -459,10 +466,16 @@ df = DataFrame( category = rand(["A", "B", "C"], 10_000_000) ) -# Send via smartsend - wrapped in a list (type is part of each tuple) -env, env_json_str = smartsend("analysis_results", [("table_data", df, "table")], broker_url="nats://localhost:4222") -# env: msg_envelope_v1 object with all metadata and payloads -# env_json_str: JSON string representation of the envelope for publishing +# Send via smartsend - wrapped in list with type +# Large payload will use link transport (HTTP fileserver) +env, env_json_str = smartsend( + "analysis_results", + [("table_data", df, "table")], + broker_url="nats://localhost:4222", + fileserver_url="http://localhost:8080" +) +# env: msg_envelope_v1 with all metadata and payloads +# env_json_str: JSON string for publishing ``` #### JavaScript (Receiver) @@ -482,19 +495,12 @@ const table = env.payloads; ```python from nats_bridge import smartsend -# Binary data wrapped in a list -binary_data = [ - ("audio_chunk", binary_buffer, "binary") -] - +# Binary data wrapped in list with type smartsend( "binary_input", - binary_data, - nats_url="nats://localhost:4222", - metadata={ - "sample_rate": 44100, - "channels": 1 - } + [("audio_chunk", binary_buffer, "binary")], + broker_url="nats://localhost:4222", + metadata={"sample_rate": 44100, "channels": 1} ) ``` @@ -558,10 +564,14 @@ function process_binary(msg) { ```julia using NATSBridge -function publish_health_status(nats_url) - # Send status wrapped in a list (type is part of each tuple) +function publish_health_status(broker_url) + # Send status wrapped in list with type status = Dict("cpu" => rand(), "memory" => rand()) - smartsend("health", [("status", status, "dictionary")], broker_url=nats_url) + env, env_json_str = smartsend( + "health", + [("status", status, "dictionary")], + broker_url=broker_url + ) sleep(5) # Every 5 seconds end ``` @@ -604,8 +614,8 @@ def handle_device_config(msg): env = smartreceive(msg) # Process configuration from payloads - for dataname, data, type in env["payloads"]: - if type == "dictionary": + for dataname, data, payload_type in env["payloads"]: + if payload_type == "dictionary": print(f"Received configuration: {data}") # Apply configuration to device if "wifi_ssid" in data: @@ -622,7 +632,7 @@ def handle_device_config(msg): smartsend( "device/response", [("config", config, "dictionary")], - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", reply_to=env.get("reply_to") ) ``` @@ -670,12 +680,14 @@ options_df = DataFrame( # Check payload size (< 1MB threshold) # Publish directly to NATS with Base64-encoded payload # Include metadata for dashboard selection context -smartsend( +env, env_json_str = smartsend( "dashboard.selection", [("options_table", options_df, "table")], - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", metadata=Dict("context" => "user_selection") ) +# env: msg_envelope_v1 with all metadata and payloads +# env_json_str: JSON string for publishing ``` **JavaScript (Receiver):** @@ -709,7 +721,6 @@ await smartsend("dashboard.response", [ **Julia (Sender/Receiver):** ```julia using NATSBridge -using DataFrames # Build chat message with mixed payloads: # - Text: direct transport (Base64) @@ -733,13 +744,15 @@ chat_message = [ ("large_document", large_file_bytes, "binary") # Large file, link transport ] -smartsend( +env, env_json_str = smartsend( "chat.room123", chat_message, broker_url="nats://localhost:4222", msg_purpose="chat", reply_to="chat.room123.responses" ) +# env: msg_envelope_v1 with all metadata and payloads +# env_json_str: JSON string for publishing ``` **JavaScript (Sender/Receiver):** -- 2.49.1 From 647cadf49712a957aeae36f1bac5003a721a2bbc Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 08:33:32 +0700 Subject: [PATCH 23/35] update --- src/NATSBridge.js | 340 ++++++++++++++++++++++++--------------- src/nats_bridge.py | 392 +++++++++++++++++++++++++++++++-------------- 2 files changed, 486 insertions(+), 246 deletions(-) diff --git a/src/NATSBridge.js b/src/NATSBridge.js index 536e123..64b1c91 100644 --- a/src/NATSBridge.js +++ b/src/NATSBridge.js @@ -98,6 +98,26 @@ function base64ToArrayBuffer(base64) { return bytes.buffer; } +// Helper: Convert Uint8Array to Base64 string +function uint8ArrayToBase64(uint8array) { + let binary = ''; + for (let i = 0; i < uint8array.byteLength; i++) { + binary += String.fromCharCode(uint8array[i]); + } + return btoa(binary); +} + +// Helper: Convert Base64 string to Uint8Array +function base64ToUint8Array(base64) { + const binaryString = atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; +} + // Helper: Serialize data based on type function _serialize_data(data, type) { /** @@ -114,39 +134,39 @@ function _serialize_data(data, type) { */ if (type === "text") { if (typeof data === 'string') { - return new TextEncoder().encode(data).buffer; + return new TextEncoder().encode(data); } else { throw new Error("Text data must be a String"); } } else if (type === "dictionary") { // JSON data - serialize directly const jsonStr = JSON.stringify(data); - return new TextEncoder().encode(jsonStr).buffer; + return new TextEncoder().encode(jsonStr); } else if (type === "table") { // Table data - convert to Arrow IPC stream (NOT IMPLEMENTED in pure JavaScript) // This would require the apache-arrow library throw new Error("Table serialization requires apache-arrow library"); } else if (type === "image") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? data : data.buffer; + return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Image data must be ArrayBuffer or Uint8Array"); } } else if (type === "audio") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? data : data.buffer; + return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Audio data must be ArrayBuffer or Uint8Array"); } } else if (type === "video") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? data : data.buffer; + return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Video data must be ArrayBuffer or Uint8Array"); } } else if (type === "binary") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? data : data.buffer; + return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Binary data must be ArrayBuffer or Uint8Array"); } @@ -171,10 +191,10 @@ function _deserialize_data(data, type, correlation_id) { */ if (type === "text") { const decoder = new TextDecoder(); - return decoder.decode(new Uint8Array(data)); + return decoder.decode(data); } else if (type === "dictionary") { const decoder = new TextDecoder(); - const jsonStr = decoder.decode(new Uint8Array(data)); + const jsonStr = decoder.decode(data); return JSON.parse(jsonStr); } else if (type === "table") { // Table data - deserialize Arrow IPC stream (NOT IMPLEMENTED in pure JavaScript) @@ -230,7 +250,7 @@ async function _upload_to_fileserver(fileserver_url, dataname, data, correlation // Create multipart form data const formData = new FormData(); - // Create a Blob from the ArrayBuffer + // Create a Blob from the Uint8Array const blob = new Blob([data], { type: "application/octet-stream" }); formData.append("file", blob, dataname); @@ -276,7 +296,7 @@ async function _fetch_with_backoff(url, max_retries, base_delay, max_delay, corr if (response.status === 200) { log_trace(correlation_id, `Successfully fetched data from ${url} on attempt ${attempt}`); const arrayBuffer = await response.arrayBuffer(); - return arrayBuffer; + return new Uint8Array(arrayBuffer); } else { throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`); } @@ -306,25 +326,26 @@ function _get_payload_bytes(data) { } } -// MessagePayload class +// MessagePayload class - matches msg_payload_v1 Julia struct class MessagePayload { /** * Represents a single payload in the message envelope + * Matches Julia's msg_payload_v1 struct * * @param {Object} options - Payload options * @param {string} options.id - ID of this payload (e.g., "uuid4") * @param {string} options.dataname - Name of this payload (e.g., "login_image") - * @param {string} options.type - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" + * @param {string} options.payload_type - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" * @param {string} options.transport - "direct" or "link" * @param {string} options.encoding - "none", "json", "base64", "arrow-ipc" * @param {number} options.size - Data size in bytes - * @param {string|ArrayBuffer} options.data - Payload data (direct) or URL (link) + * @param {string|Uint8Array} options.data - Payload data (Uint8Array for direct, URL string for link) * @param {Object} options.metadata - Metadata for this payload */ constructor(options) { this.id = options.id || uuid4(); this.dataname = options.dataname; - this.type = options.type; + this.payload_type = options.payload_type; this.transport = options.transport; this.encoding = options.encoding; this.size = options.size; @@ -332,27 +353,27 @@ class MessagePayload { this.metadata = options.metadata || {}; } - // Convert to JSON object + // Convert to JSON object - uses snake_case to match Julia API toJSON() { const obj = { id: this.id, dataname: this.dataname, - type: this.type, + payload_type: this.payload_type, transport: this.transport, encoding: this.encoding, size: this.size }; // Include data based on transport type - if (this.transport === "direct" && this.data !== null) { + if (this.transport === "direct" && this.data !== null && this.data !== undefined) { if (this.encoding === "base64" || this.encoding === "json") { obj.data = this.data; } else { // For other encodings, use base64 const payloadBytes = _get_payload_bytes(this.data); - obj.data = arrayBufferToBase64(payloadBytes); + obj.data = uint8ArrayToBase64(payloadBytes); } - } else if (this.transport === "link" && this.data !== null) { + } else if (this.transport === "link" && this.data !== null && this.data !== undefined) { // For link transport, data is a URL string obj.data = this.data; } @@ -365,59 +386,60 @@ class MessagePayload { } } -// MessageEnvelope class +// MessageEnvelope class - matches msg_envelope_v1 Julia struct class MessageEnvelope { /** * Represents the message envelope containing metadata and payloads + * Matches Julia's msg_envelope_v1 struct * * @param {Object} options - Envelope options - * @param {string} options.sendTo - Topic/subject the sender sends to - * @param {Array} options.payloads - Array of payloads - * @param {string} options.correlationId - Unique identifier to track messages - * @param {string} options.msgId - This message id + * @param {string} options.correlation_id - Unique identifier to track messages + * @param {string} options.msg_id - This message id * @param {string} options.timestamp - Message published timestamp - * @param {string} options.msgPurpose - Purpose of this message - * @param {string} options.senderName - Name of the sender - * @param {string} options.senderId - UUID of the sender - * @param {string} options.receiverName - Name of the receiver - * @param {string} options.receiverId - UUID of the receiver - * @param {string} options.replyTo - Topic to reply to - * @param {string} options.replyToMsgId - Message id this message is replying to - * @param {string} options.brokerURL - NATS server address + * @param {string} options.send_to - Topic/subject the sender sends to + * @param {string} options.msg_purpose - Purpose of this message + * @param {string} options.sender_name - Name of the sender + * @param {string} options.sender_id - UUID of the sender + * @param {string} options.receiver_name - Name of the receiver + * @param {string} options.receiver_id - UUID of the receiver + * @param {string} options.reply_to - Topic to reply to + * @param {string} options.reply_to_msg_id - Message id this message is replying to + * @param {string} options.broker_url - NATS server address * @param {Object} options.metadata - Metadata for the envelope + * @param {Array} options.payloads - Array of payloads */ constructor(options) { - this.correlationId = options.correlationId || uuid4(); - this.msgId = options.msgId || uuid4(); + this.correlation_id = options.correlation_id || uuid4(); + this.msg_id = options.msg_id || uuid4(); this.timestamp = options.timestamp || new Date().toISOString(); - this.sendTo = options.sendTo; - this.msgPurpose = options.msgPurpose || ""; - this.senderName = options.senderName || ""; - this.senderId = options.senderId || uuid4(); - this.receiverName = options.receiverName || ""; - this.receiverId = options.receiverId || ""; - this.replyTo = options.replyTo || ""; - this.replyToMsgId = options.replyToMsgId || ""; - this.brokerURL = options.brokerURL || DEFAULT_NATS_URL; + this.send_to = options.send_to; + this.msg_purpose = options.msg_purpose || ""; + this.sender_name = options.sender_name || ""; + this.sender_id = options.sender_id || uuid4(); + this.receiver_name = options.receiver_name || ""; + this.receiver_id = options.receiver_id || ""; + this.reply_to = options.reply_to || ""; + this.reply_to_msg_id = options.reply_to_msg_id || ""; + this.broker_url = options.broker_url || DEFAULT_NATS_URL; this.metadata = options.metadata || {}; this.payloads = options.payloads || []; } - // Convert to JSON string + // Convert to JSON object - uses snake_case to match Julia API toJSON() { const obj = { - correlationId: this.correlationId, - msgId: this.msgId, + correlation_id: this.correlation_id, + msg_id: this.msg_id, timestamp: this.timestamp, - sendTo: this.sendTo, - msgPurpose: this.msgPurpose, - senderName: this.senderName, - senderId: this.senderId, - receiverName: this.receiverName, - receiverId: this.receiverId, - replyTo: this.replyTo, - replyToMsgId: this.replyToMsgId, - brokerURL: this.brokerURL + send_to: this.send_to, + msg_purpose: this.msg_purpose, + sender_name: this.sender_name, + sender_id: this.sender_id, + receiver_name: this.receiver_name, + receiver_id: this.receiver_id, + reply_to: this.reply_to, + reply_to_msg_id: this.reply_to_msg_id, + broker_url: this.broker_url }; if (Object.keys(this.metadata).length > 0) { @@ -437,7 +459,7 @@ class MessageEnvelope { } } -// SmartSend function +// SmartSend function - matches Julia smartsend signature and behavior async function smartsend(subject, data, options = {}) { /** * Send data either directly via NATS or via a fileserver URL, depending on payload size @@ -447,42 +469,42 @@ async function smartsend(subject, data, options = {}) { * Otherwise, it uploads the data to a fileserver and publishes only the download URL over NATS. * * @param {string} subject - NATS subject to publish the message to - * @param {Array} data - List of {dataname, data, type} objects to send + * @param {Array} data - List of {dataname, data, type} objects to send (must be a list, even for single payload) * @param {Object} options - Additional options - * @param {string} options.natsUrl - URL of the NATS server (default: "nats://localhost:4222") - * @param {string} options.fileserverUrl - Base URL of the file server (default: "http://localhost:8080") - * @param {Function} options.fileserverUploadHandler - Function to handle fileserver uploads - * @param {number} options.sizeThreshold - Threshold in bytes separating direct vs link transport (default: 1MB) - * @param {string} options.correlationId - Optional correlation ID for tracing - * @param {string} options.msgPurpose - Purpose of the message (default: "chat") - * @param {string} options.senderName - Name of the sender (default: "NATSBridge") - * @param {string} options.receiverName - Name of the receiver (default: "") - * @param {string} options.receiverId - UUID of the receiver (default: "") - * @param {string} options.replyTo - Topic to reply to (default: "") - * @param {string} options.replyToMsgId - Message ID this message is replying to (default: "") - * @param {boolean} options.isPublish - Whether to automatically publish the message to NATS (default: true) + * @param {string} options.broker_url - URL of the NATS server (default: "nats://localhost:4222") + * @param {string} options.fileserver_url - Base URL of the file server (default: "http://localhost:8080") + * @param {Function} options.fileserver_upload_handler - Function to handle fileserver uploads + * @param {number} options.size_threshold - Threshold in bytes separating direct vs link transport (default: 1MB) + * @param {string} options.correlation_id - Optional correlation ID for tracing + * @param {string} options.msg_purpose - Purpose of the message (default: "chat") + * @param {string} options.sender_name - Name of the sender (default: "NATSBridge") + * @param {string} options.receiver_name - Name of the receiver (default: "") + * @param {string} options.receiver_id - UUID of the receiver (default: "") + * @param {string} options.reply_to - Topic to reply to (default: "") + * @param {string} options.reply_to_msg_id - Message ID this message is replying to (default: "") + * @param {boolean} options.is_publish - Whether to automatically publish the message to NATS (default: true) * - * @returns {Promise} - An object with { env: MessageEnvelope, env_json_str: string } + * @returns {Promise} - A tuple-like object with { env: MessageEnvelope, env_json_str: string } */ const { - natsUrl = DEFAULT_NATS_URL, - fileserverUrl = DEFAULT_FILESERVER_URL, - fileserverUploadHandler = _upload_to_fileserver, - sizeThreshold = DEFAULT_SIZE_THRESHOLD, - correlationId = uuid4(), - msgPurpose = "chat", - senderName = "NATSBridge", - receiverName = "", - receiverId = "", - replyTo = "", - replyToMsgId = "", - isPublish = true // Whether to automatically publish the message to NATS + broker_url = DEFAULT_NATS_URL, + fileserver_url = DEFAULT_FILESERVER_URL, + fileserver_upload_handler = _upload_to_fileserver, + size_threshold = DEFAULT_SIZE_THRESHOLD, + correlation_id = uuid4(), + msg_purpose = "chat", + sender_name = "NATSBridge", + receiver_name = "", + receiver_id = "", + reply_to = "", + reply_to_msg_id = "", + is_publish = true // Whether to automatically publish the message to NATS } = options; - log_trace(correlationId, `Starting smartsend for subject: ${subject}`); + log_trace(correlation_id, `Starting smartsend for subject: ${subject}`); // Generate message metadata - const msgId = uuid4(); + const msg_id = uuid4(); // Process each payload in the list const payloads = []; @@ -496,18 +518,18 @@ async function smartsend(subject, data, options = {}) { const payloadBytes = _serialize_data(payloadData, payloadType); const payloadSize = payloadBytes.byteLength; - log_trace(correlationId, `Serialized payload '${dataname}' (type: ${payloadType}) size: ${payloadSize} bytes`); + log_trace(correlation_id, `Serialized payload '${dataname}' (payload_type: ${payloadType}) size: ${payloadSize} bytes`); // Decision: Direct vs Link - if (payloadSize < sizeThreshold) { + if (payloadSize < size_threshold) { // Direct path - Base64 encode and send via NATS - const payloadB64 = arrayBufferToBase64(payloadBytes); - log_trace(correlationId, `Using direct transport for ${payloadSize} bytes`); + const payloadB64 = uint8ArrayToBase64(payloadBytes); + log_trace(correlation_id, `Using direct transport for ${payloadSize} bytes`); // Create MessagePayload for direct transport const payloadObj = new MessagePayload({ dataname: dataname, - type: payloadType, + payload_type: payloadType, transport: "direct", encoding: "base64", size: payloadSize, @@ -517,22 +539,22 @@ async function smartsend(subject, data, options = {}) { payloads.push(payloadObj); } else { // Link path - Upload to HTTP server, send URL via NATS - log_trace(correlationId, `Using link transport, uploading to fileserver`); + log_trace(correlation_id, `Using link transport, uploading to fileserver`); // Upload to HTTP server - const response = await fileserverUploadHandler(fileserverUrl, dataname, payloadBytes, correlationId); + const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes, correlation_id); if (response.status !== 200) { throw new Error(`Failed to upload data to fileserver: ${response.status}`); } const url = response.url; - log_trace(correlationId, `Uploaded to URL: ${url}`); + log_trace(correlation_id, `Uploaded to URL: ${url}`); // Create MessagePayload for link transport const payloadObj = new MessagePayload({ dataname: dataname, - type: payloadType, + payload_type: payloadType, transport: "link", encoding: "none", size: payloadSize, @@ -545,16 +567,16 @@ async function smartsend(subject, data, options = {}) { // Create MessageEnvelope with all payloads const env = new MessageEnvelope({ - correlationId: correlationId, - msgId: msgId, - sendTo: subject, - msgPurpose: msgPurpose, - senderName: senderName, - receiverName: receiverName, - receiverId: receiverId, - replyTo: replyTo, - replyToMsgId: replyToMsgId, - brokerURL: natsUrl, + correlation_id: correlation_id, + msg_id: msg_id, + send_to: subject, + msg_purpose: msg_purpose, + sender_name: sender_name, + receiver_name: receiver_name, + receiver_id: receiver_id, + reply_to: reply_to, + reply_to_msg_id: reply_to_msg_id, + broker_url: broker_url, payloads: payloads }); @@ -562,11 +584,11 @@ async function smartsend(subject, data, options = {}) { const env_json_str = env.toString(); // Publish to NATS if isPublish is true - if (isPublish) { - await publish_message(natsUrl, subject, env_json_str, correlationId); + if (is_publish) { + await publish_message(broker_url, subject, env_json_str, correlation_id); } - // Return both envelope and JSON string (tuple-like structure) + // Return both envelope and JSON string (tuple-like structure, matching Julia API) return { env: env, env_json_str: env_json_str @@ -574,11 +596,11 @@ async function smartsend(subject, data, options = {}) { } // Helper: Publish message to NATS -async function publish_message(natsUrl, subject, message, correlation_id) { +async function publish_message(broker_url, subject, message, correlation_id) { /** * Publish a message to a NATS subject with proper connection management * - * @param {string} natsUrl - NATS server URL + * @param {string} broker_url - NATS server URL * @param {string} subject - NATS subject to publish to * @param {string} message - JSON message to publish * @param {string} correlation_id - Correlation ID for logging @@ -591,7 +613,7 @@ async function publish_message(natsUrl, subject, message, correlation_id) { // Example with nats.js: // import { connect } from 'nats'; - // const nc = await connect({ servers: [natsUrl] }); + // const nc = await connect({ servers: [broker_url] }); // await nc.publish(subject, message); // nc.close(); @@ -599,7 +621,7 @@ async function publish_message(natsUrl, subject, message, correlation_id) { console.log(`[NATS PUBLISH] Subject: ${subject}, Message: ${message.substring(0, 100)}...`); } -// SmartReceive function +// SmartReceive function - matches Julia smartreceive signature and behavior async function smartreceive(msg, options = {}) { /** * Receive and process messages from NATS @@ -609,25 +631,25 @@ async function smartreceive(msg, options = {}) { * * @param {Object} msg - NATS message object with payload property * @param {Object} options - Additional options - * @param {Function} options.fileserverDownloadHandler - Function to handle downloading data from file server URLs - * @param {number} options.maxRetries - Maximum retry attempts for fetching URL (default: 5) - * @param {number} options.baseDelay - Initial delay for exponential backoff in ms (default: 100) - * @param {number} options.maxDelay - Maximum delay for exponential backoff in ms (default: 5000) + * @param {Function} options.fileserver_download_handler - Function to handle downloading data from file server URLs + * @param {number} options.max_retries - Maximum retry attempts for fetching URL (default: 5) + * @param {number} options.base_delay - Initial delay for exponential backoff in ms (default: 100) + * @param {number} options.max_delay - Maximum delay for exponential backoff in ms (default: 5000) * - * @returns {Promise} - Envelope dictionary with metadata and payloads field containing list of {dataname, data, type} objects + * @returns {Promise} - JSON object of envelope with payloads field containing list of {dataname, data, type} tuples */ const { - fileserverDownloadHandler = _fetch_with_backoff, - maxRetries = 5, - baseDelay = 100, - maxDelay = 5000 + fileserver_download_handler = _fetch_with_backoff, + max_retries = 5, + base_delay = 100, + max_delay = 5000 } = options; // Parse the JSON envelope const jsonStr = typeof msg.payload === 'string' ? msg.payload : new TextDecoder().decode(msg.payload); const json_data = JSON.parse(jsonStr); - log_trace(json_data.correlationId, `Processing received message`); + log_trace(json_data.correlation_id, `Processing received message`); // Process all payloads in the envelope const payloads_list = []; @@ -642,32 +664,32 @@ async function smartreceive(msg, options = {}) { if (transport === "direct") { // Direct transport - payload is in the message - log_trace(json_data.correlationId, `Direct transport - decoding payload '${dataname}'`); + log_trace(json_data.correlation_id, `Direct transport - decoding payload '${dataname}'`); // Extract base64 payload from the payload const payload_b64 = payload.data; // Decode Base64 payload - const payload_bytes = base64ToArrayBuffer(payload_b64); + const payload_bytes = base64ToUint8Array(payload_b64); // Deserialize based on type - const data_type = payload.type; - const data = _deserialize_data(payload_bytes, data_type, json_data.correlationId); + const data_type = payload.payload_type; + const data = _deserialize_data(payload_bytes, data_type, json_data.correlation_id); payloads_list.push({ dataname, data, type: data_type }); } else if (transport === "link") { // Link transport - payload is at URL const url = payload.data; - log_trace(json_data.correlationId, `Link transport - fetching '${dataname}' from URL: ${url}`); + log_trace(json_data.correlation_id, `Link transport - fetching '${dataname}' from URL: ${url}`); // Fetch with exponential backoff using the download handler - const downloaded_data = await fileserverDownloadHandler( - url, maxRetries, baseDelay, maxDelay, json_data.correlationId + const downloaded_data = await fileserver_download_handler( + url, max_retries, base_delay, max_delay, json_data.correlation_id ); // Deserialize based on type - const data_type = payload.type; - const data = _deserialize_data(downloaded_data, data_type, json_data.correlationId); + const data_type = payload.payload_type; + const data = _deserialize_data(downloaded_data, data_type, json_data.correlation_id); payloads_list.push({ dataname, data, type: data_type }); } else { @@ -676,11 +698,69 @@ async function smartreceive(msg, options = {}) { } // Replace payloads array with the processed list of {dataname, data, type} tuples + // This matches Julia's smartreceive return format json_data.payloads = payloads_list; return json_data; } +// plik_oneshot_upload - matches Julia plik_oneshot_upload function +async function plik_oneshot_upload(file_server_url, dataname, data) { + /** + * Upload a single file to a plik server using one-shot mode + * This function uploads raw byte array to a plik server in one-shot mode (no upload session). + * It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, + * retrieves an upload ID and token, then uploads the file data as multipart form data using the token. + * + * @param {string} file_server_url - Base URL of the plik server (e.g., "http://localhost:8080") + * @param {string} dataname - Name of the file being uploaded + * @param {Uint8Array} data - Raw byte data of the file content + * @returns {Promise} - Dictionary with keys: status, uploadid, fileid, url + */ + + // Step 1: Get upload ID and token + const url_getUploadID = `${file_server_url}/upload`; + const headers = { "Content-Type": "application/json" }; + const body = JSON.stringify({ OneShot: true }); + + let http_response = await fetch(url_getUploadID, { + method: "POST", + headers: headers, + body: body + }); + + const response_json = await http_response.json(); + const uploadid = response_json.id; + const uploadtoken = response_json.uploadToken; + + // Step 2: Upload file data + const url_upload = `${file_server_url}/file/${uploadid}`; + + // Create multipart form data + const formData = new FormData(); + const blob = new Blob([data], { type: "application/octet-stream" }); + formData.append("file", blob, dataname); + + http_response = await fetch(url_upload, { + method: "POST", + headers: { "X-UploadToken": uploadtoken }, + body: formData + }); + + const fileResponseJson = await http_response.json(); + const fileid = fileResponseJson.id; + + // URL of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" + const url = `${file_server_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; + + return { + status: http_response.status, + uploadid: uploadid, + fileid: fileid, + url: url + }; +} + // Export for Node.js if (typeof module !== 'undefined' && module.exports) { module.exports = { @@ -692,6 +772,7 @@ if (typeof module !== 'undefined' && module.exports) { _deserialize_data, _fetch_with_backoff, _upload_to_fileserver, + plik_oneshot_upload, DEFAULT_SIZE_THRESHOLD, DEFAULT_NATS_URL, DEFAULT_FILESERVER_URL, @@ -711,6 +792,7 @@ if (typeof window !== 'undefined') { _deserialize_data, _fetch_with_backoff, _upload_to_fileserver, + plik_oneshot_upload, DEFAULT_SIZE_THRESHOLD, DEFAULT_NATS_URL, DEFAULT_FILESERVER_URL, diff --git a/src/nats_bridge.py b/src/nats_bridge.py index 7e73473..2b4c457 100644 --- a/src/nats_bridge.py +++ b/src/nats_bridge.py @@ -1,45 +1,60 @@ """ -Micropython NATS Bridge - Bi-Directional Data Bridge for Micropython +Python NATS Bridge - Bi-Directional Data Bridge This module provides functionality for sending and receiving data over NATS using the Claim-Check pattern for large payloads. Supported types: "text", "dictionary", "table", "image", "audio", "video", "binary" + +Multi-Payload Support (Standard API): +The system uses a standardized list-of-tuples format for all payload operations. +Even when sending a single payload, the user must wrap it in a list. + +API Standard: + # Input format for smartsend (always a list of tuples with type info) + [(dataname1, data1, type1), (dataname2, data2, type2), ...] + + # Output format for smartreceive (always returns a list of tuples) + [(dataname1, data1, type1), (dataname2, data2, type2), ...] """ import json -import random import time -import usocket -import uselect -import ustruct import uuid -try: - import ussl - HAS_SSL = True -except ImportError: - HAS_SSL = False - # Constants DEFAULT_SIZE_THRESHOLD = 1000000 # 1MB - threshold for switching from direct to link transport -DEFAULT_NATS_URL = "nats://localhost:4222" +DEFAULT_BROKER_URL = "nats://localhost:4222" DEFAULT_FILESERVER_URL = "http://localhost:8080" # ============================================= 100 ============================================== # class MessagePayload: - """Internal message payload structure representing a single payload within a NATS message envelope.""" + """Internal message payload structure representing a single payload within a NATS message envelope. - def __init__(self, data, msg_type, id="", dataname="", transport="direct", + This structure represents a single payload within a NATS message envelope. + It supports both direct transport (base64-encoded data) and link transport (URL-based). + + Attributes: + id: Unique identifier for this payload (e.g., "uuid4") + dataname: Name of the payload (e.g., "login_image") + payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") + transport: Transport method ("direct" or "link") + encoding: Encoding method ("none", "json", "base64", "arrow-ipc") + size: Size of the payload in bytes + data: Payload data (bytes for direct, URL for link) + metadata: Optional metadata dictionary + """ + + def __init__(self, data, payload_type, id="", dataname="", transport="direct", encoding="none", size=0, metadata=None): """ Initialize a MessagePayload. Args: - data: Payload data (bytes for direct, URL string for link) - msg_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") + data: Payload data (base64 string for direct, URL string for link) + payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") id: Unique identifier for this payload (auto-generated if empty) dataname: Name of the payload (auto-generated UUID if empty) transport: Transport method ("direct" or "link") @@ -49,7 +64,7 @@ class MessagePayload: """ self.id = id if id else self._generate_uuid() self.dataname = dataname if dataname else self._generate_uuid() - self.type = msg_type + self.payload_type = payload_type self.transport = transport self.encoding = encoding self.size = size @@ -65,7 +80,7 @@ class MessagePayload: payload_dict = { "id": self.id, "dataname": self.dataname, - "type": self.type, + "payload_type": self.payload_type, "transport": self.transport, "encoding": self.encoding, "size": self.size, @@ -152,20 +167,24 @@ class MessageEnvelope: return "2026-02-21T" + time.strftime("%H:%M:%S", time.localtime()) def to_json(self): - """Convert envelope to JSON string.""" + """Convert envelope to JSON string. + + Returns: + str: JSON string representation of the envelope using snake_case field names + """ obj = { - "correlationId": self.correlation_id, - "msgId": self.msg_id, + "correlation_id": self.correlation_id, + "msg_id": self.msg_id, "timestamp": self.timestamp, - "sendTo": self.send_to, - "msgPurpose": self.msg_purpose, - "senderName": self.sender_name, - "senderId": self.sender_id, - "receiverName": self.receiver_name, - "receiverId": self.receiver_id, - "replyTo": self.reply_to, - "replyToMsgId": self.reply_to_msg_id, - "brokerURL": self.broker_url + "send_to": self.send_to, + "msg_purpose": self.msg_purpose, + "sender_name": self.sender_name, + "sender_id": self.sender_id, + "receiver_name": self.receiver_name, + "receiver_id": self.receiver_id, + "reply_to": self.reply_to, + "reply_to_msg_id": self.reply_to_msg_id, + "broker_url": self.broker_url } # Include metadata if not empty @@ -188,68 +207,126 @@ def log_trace(correlation_id, message): print("[{}] [Correlation: {}] {}".format(timestamp, correlation_id, message)) -def _serialize_data(data, msg_type): +def _serialize_data(data, payload_type): """Serialize data according to specified format. + This function serializes arbitrary data into a binary representation based on the specified type. + It supports multiple serialization formats for different data types. + Args: data: Data to serialize - msg_type: Target format ("text", "dictionary", "table", "image", "audio", "video", "binary") + - "text": String + - "dictionary": JSON-serializable dict + - "table": Tabular data (pandas DataFrame or list of dicts) + - "image", "audio", "video", "binary": bytes + payload_type: Target format ("text", "dictionary", "table", "image", "audio", "video", "binary") Returns: bytes: Binary representation of the serialized data + + Example: + >>> text_bytes = _serialize_data("Hello World", "text") + >>> json_bytes = _serialize_data({"key": "value"}, "dictionary") + >>> table_bytes = _serialize_data([{"id": 1, "name": "Alice"}], "table") """ - if msg_type == "text": + if payload_type == "text": if isinstance(data, str): return data.encode('utf-8') else: raise ValueError("Text data must be a string") - elif msg_type == "dictionary": + elif payload_type == "dictionary": if isinstance(data, dict): json_str = json.dumps(data) return json_str.encode('utf-8') else: raise ValueError("Dictionary data must be a dict") - elif msg_type in ("image", "audio", "video", "binary"): + elif payload_type == "table": + # Support pandas DataFrame or list of dicts + try: + import pandas as pd + if isinstance(data, pd.DataFrame): + # Convert DataFrame to JSON and then to bytes + json_str = data.to_json(orient='records', force_ascii=False) + return json_str.encode('utf-8') + elif isinstance(data, list) and len(data) > 0 and isinstance(data[0], dict): + # List of dicts + json_str = json.dumps(data) + return json_str.encode('utf-8') + else: + raise ValueError("Table data must be a pandas DataFrame or list of dicts") + except ImportError: + # Fallback: if pandas not available, treat as list of dicts + if isinstance(data, list): + json_str = json.dumps(data) + return json_str.encode('utf-8') + else: + raise ValueError("Table data requires pandas DataFrame or list of dicts (pandas not available)") + + elif payload_type in ("image", "audio", "video", "binary"): if isinstance(data, bytes): return data else: - raise ValueError("{} data must be bytes".format(msg_type.capitalize())) + raise ValueError("{} data must be bytes".format(payload_type.capitalize())) else: - raise ValueError("Unknown type: {}".format(msg_type)) + raise ValueError("Unknown payload_type: {}".format(payload_type)) -def _deserialize_data(data_bytes, msg_type, correlation_id): +def _deserialize_data(data_bytes, payload_type, correlation_id): """Deserialize bytes to data based on type. + This function converts serialized bytes back to Python data based on type. + It handles "text" (string), "dictionary" (JSON deserialization), "table" (JSON deserialization), + "image" (binary data), "audio" (binary data), "video" (binary data), and "binary" (binary data). + Args: data_bytes: Serialized data as bytes - msg_type: Data type ("text", "dictionary", "table", "image", "audio", "video", "binary") + payload_type: Data type ("text", "dictionary", "table", "image", "audio", "video", "binary") correlation_id: Correlation ID for logging Returns: - Deserialized data + Deserialized data: + - "text": str + - "dictionary": dict + - "table": list of dicts (or pandas DataFrame if available) + - "image", "audio", "video", "binary": bytes + + Example: + >>> text_data = _deserialize_data(b"Hello", "text", "corr_id") + >>> json_data = _deserialize_data(b'{"key": "value"}', "dictionary", "corr_id") + >>> table_data = _deserialize_data(b'[{"id": 1}]', "table", "corr_id") """ - if msg_type == "text": + if payload_type == "text": return data_bytes.decode('utf-8') - elif msg_type == "dictionary": + elif payload_type == "dictionary": json_str = data_bytes.decode('utf-8') return json.loads(json_str) - elif msg_type in ("image", "audio", "video", "binary"): + elif payload_type == "table": + # Deserialize table data (JSON format) + json_str = data_bytes.decode('utf-8') + table_data = json.loads(json_str) + # If pandas is available, try to convert to DataFrame + try: + import pandas as pd + return pd.DataFrame(table_data) + except ImportError: + return table_data + + elif payload_type in ("image", "audio", "video", "binary"): return data_bytes else: - raise ValueError("Unknown type: {}".format(msg_type)) + raise ValueError("Unknown payload_type: {}".format(payload_type)) class NATSConnection: - """Simple NATS connection for Micropython.""" + """Simple NATS connection for Python and Micropython.""" - def __init__(self, url=DEFAULT_NATS_URL): + def __init__(self, url=DEFAULT_BROKER_URL): """Initialize NATS connection. Args: @@ -276,9 +353,19 @@ class NATSConnection: def connect(self): """Connect to NATS server.""" - addr = usocket.getaddrinfo(self.host, self.port)[0][-1] - self.conn = usocket.socket() - self.conn.connect(addr) + # Use socket for both Python and Micropython + try: + import socket + addr = socket.getaddrinfo(self.host, self.port)[0][-1] + self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.conn.connect(addr) + except NameError: + # Micropython fallback + import usocket + addr = usocket.getaddrinfo(self.host, self.port)[0][-1] + self.conn = usocket.socket() + self.conn.connect(addr) + log_trace("", "Connected to NATS server at {}:{}".format(self.host, self.port)) def publish(self, subject, message): @@ -294,7 +381,15 @@ class NATSConnection: # Simple NATS protocol implementation msg = "PUB {} {}\r\n".format(subject, len(message)) msg = msg.encode('utf-8') + message + b"\r\n" - self.conn.send(msg) + + try: + import socket + self.conn.send(msg) + except NameError: + # Micropython fallback + import usocket + self.conn.send(msg) + log_trace("", "Message published to {}".format(subject)) def subscribe(self, subject, callback): @@ -335,11 +430,14 @@ class NATSConnection: def _fetch_with_backoff(url, max_retries=5, base_delay=100, max_delay=5000, correlation_id=""): """Fetch data from URL with exponential backoff. + This function retrieves data from a URL with retry logic using + exponential backoff to handle transient failures. + Args: url: URL to fetch from - max_retries: Maximum number of retry attempts - base_delay: Initial delay in milliseconds - max_delay: Maximum delay in milliseconds + max_retries: Maximum number of retry attempts (default: 5) + base_delay: Initial delay in milliseconds (default: 100) + max_delay: Maximum delay in milliseconds (default: 5000) correlation_id: Correlation ID for logging Returns: @@ -347,33 +445,54 @@ def _fetch_with_backoff(url, max_retries=5, base_delay=100, max_delay=5000, corr Raises: Exception: If all retry attempts fail + + Example: + >>> data = _fetch_with_backoff("http://example.com/file.zip", 5, 100, 5000, "corr_id") """ delay = base_delay for attempt in range(1, max_retries + 1): try: # Simple HTTP GET request - # This is a simplified implementation - # For production, you'd want a proper HTTP client - import urequests - response = urequests.get(url) - if response.status_code == 200: + # Try urequests for Micropython first, then requests for Python + try: + import urequests + response = urequests.get(url) + status_code = response.status_code + content = response.content + except ImportError: + try: + import requests + response = requests.get(url) + response.raise_for_status() + status_code = response.status_code + content = response.content + except ImportError: + raise Exception("No HTTP library available (urequests or requests)") + + if status_code == 200: log_trace(correlation_id, "Successfully fetched data from {} on attempt {}".format(url, attempt)) - return response.content + return content else: - raise Exception("Failed to fetch: {}".format(response.status_code)) + raise Exception("Failed to fetch: {}".format(status_code)) except Exception as e: log_trace(correlation_id, "Attempt {} failed: {}".format(attempt, str(e))) if attempt < max_retries: time.sleep(delay / 1000.0) delay = min(delay * 2, max_delay) + + raise Exception("Failed to fetch data after {} attempts".format(max_retries)) -def plik_oneshot_upload(file_server_url, filename, data): +def plik_oneshot_upload(fileserver_url, dataname, data): """Upload a single file to a plik server using one-shot mode. + This function uploads raw byte data to a plik server in one-shot mode (no upload session). + It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, + retrieves an upload ID and token, then uploads the file data as multipart form data using the token. + Args: - file_server_url: Base URL of the plik server - filename: Name of the file being uploaded + fileserver_url: Base URL of the plik server (e.g., "http://localhost:8080") + dataname: Name of the file being uploaded data: Raw byte data of the file content Returns: @@ -382,23 +501,31 @@ def plik_oneshot_upload(file_server_url, filename, data): - "uploadid": ID of the one-shot upload session - "fileid": ID of the uploaded file within the session - "url": Full URL to download the uploaded file + + Example: + >>> result = plik_oneshot_upload("http://localhost:8080", "test.txt", b"hello world") + >>> result["status"], result["uploadid"], result["fileid"], result["url"] """ - import urequests import json + try: + import urequests + except ImportError: + import requests as urequests + # Get upload ID - url_get_upload_id = "{}/upload".format(file_server_url) + url_get_upload_id = "{}/upload".format(fileserver_url) headers = {"Content-Type": "application/json"} body = json.dumps({"OneShot": True}) response = urequests.post(url_get_upload_id, headers=headers, data=body) - response_json = json.loads(response.content) + response_json = json.loads(response.text if hasattr(response, 'text') else response.content) uploadid = response_json.get("id") uploadtoken = response_json.get("uploadToken") # Upload file - url_upload = "{}/file/{}".format(file_server_url, uploadid) + url_upload = "{}/file/{}".format(fileserver_url, uploadid) headers = {"X-UploadToken": uploadtoken} # For Micropython, we need to construct the multipart form data manually @@ -407,7 +534,7 @@ def plik_oneshot_upload(file_server_url, filename, data): # Create multipart body part1 = "--{}\r\n".format(boundary) - part1 += "Content-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n".format(filename) + part1 += "Content-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n".format(dataname) part1 += "Content-Type: application/octet-stream\r\n\r\n" part1_bytes = part1.encode('utf-8') @@ -421,10 +548,10 @@ def plik_oneshot_upload(file_server_url, filename, data): content_type = "multipart/form-data; boundary={}".format(boundary) response = urequests.post(url_upload, headers={"Content-Type": content_type}, data=full_body) - response_json = json.loads(response.content) + response_json = json.loads(response.text if hasattr(response, 'text') else response.content) fileid = response_json.get("id") - url = "{}/file/{}/{}".format(file_server_url, uploadid, filename) + url = "{}/file/{}/{}".format(fileserver_url, uploadid, dataname) return { "status": response.status_code, @@ -434,7 +561,7 @@ def plik_oneshot_upload(file_server_url, filename, data): } -def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_FILESERVER_URL, +def smartsend(subject, data, broker_url=DEFAULT_BROKER_URL, fileserver_url=DEFAULT_FILESERVER_URL, fileserver_upload_handler=plik_oneshot_upload, size_threshold=DEFAULT_SIZE_THRESHOLD, correlation_id=None, msg_purpose="chat", sender_name="NATSBridge", receiver_name="", receiver_id="", reply_to="", reply_to_msg_id="", is_publish=True): @@ -447,27 +574,38 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F Args: subject: NATS subject to publish the message to - data: List of (dataname, data, type) tuples to send - nats_url: URL of the NATS server + data: List of (dataname, data, payload_type) tuples to send + - dataname: Name of the payload + - data: The actual data to send + - payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") + broker_url: URL of the NATS server fileserver_url: URL of the HTTP file server - fileserver_upload_handler: Function to handle fileserver uploads - size_threshold: Threshold in bytes separating direct vs link transport - correlation_id: Optional correlation ID for tracing - msg_purpose: Purpose of the message + fileserver_upload_handler: Function to handle fileserver uploads (must return dict with "status", "uploadid", "fileid", "url" keys) + size_threshold: Threshold in bytes separating direct vs link transport (default: 1MB) + correlation_id: Optional correlation ID for tracing; if None, a UUID is generated + msg_purpose: Purpose of the message ("ACK", "NACK", "updateStatus", "shutdown", "chat", etc.) sender_name: Name of the sender - receiver_name: Name of the receiver - receiver_id: UUID of the receiver - reply_to: Topic to reply to + receiver_name: Name of the receiver (empty string means broadcast) + receiver_id: UUID of the receiver (empty string means broadcast) + reply_to: Topic to reply to (empty string if no reply expected) reply_to_msg_id: Message ID this message is replying to is_publish: Whether to automatically publish the message to NATS (default: True) + - When True: message is published to NATS + - When False: returns envelope and JSON string without publishing Returns: tuple: (env, env_json_str) where: - env: MessageEnvelope object with all metadata and payloads - env_json_str: JSON string representation of the envelope for publishing + + Example: + >>> data = [("message", "Hello World!", "text")] + >>> env, env_json_str = smartsend("/test", data) + >>> # env: MessageEnvelope with all metadata and payloads + >>> # env_json_str: JSON string for publishing """ # Generate correlation ID if not provided - cid = correlation_id if correlation_id else str(uuid.uuid4()) + cid = correlation_id if correlation_id is not None else str(uuid.uuid4()) log_trace(cid, "Starting smartsend for subject: {}".format(subject)) @@ -482,16 +620,19 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F payload_bytes = _serialize_data(payload_data, payload_type) payload_size = len(payload_bytes) - log_trace(cid, "Serialized payload '{}' (type: {}) size: {} bytes".format( + log_trace(cid, "Serialized payload '{}' (payload_type: {}) size: {} bytes".format( dataname, payload_type, payload_size)) # Decision: Direct vs Link if payload_size < size_threshold: # Direct path - Base64 encode and send via NATS - payload_b64 = _serialize_data(payload_bytes, "binary") # Already bytes # Convert to base64 string for JSON - import ubinascii - payload_b64_str = ubinascii.b2a_base64(payload_bytes).decode('utf-8').strip() + try: + import ubinascii + payload_b64_str = ubinascii.b2a_base64(payload_bytes).decode('utf-8').strip() + except ImportError: + import base64 + payload_b64_str = base64.b64encode(payload_bytes).decode('utf-8') log_trace(cid, "Using direct transport for {} bytes".format(payload_size)) @@ -514,10 +655,10 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F # Upload to HTTP server response = fileserver_upload_handler(fileserver_url, dataname, payload_bytes) - if response["status"] != 200: - raise Exception("Failed to upload data to fileserver: {}".format(response["status"])) + if response.get("status") != 200: + raise Exception("Failed to upload data to fileserver: {}".format(response.get("status"))) - url = response["url"] + url = response.get("url") log_trace(cid, "Uploaded to URL: {}".format(url)) # Create MessagePayload for link transport @@ -546,7 +687,7 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F receiver_id=receiver_id, reply_to=reply_to, reply_to_msg_id=reply_to_msg_id, - broker_url=nats_url, + broker_url=broker_url, metadata={} ) @@ -554,7 +695,7 @@ def smartsend(subject, data, nats_url=DEFAULT_NATS_URL, fileserver_url=DEFAULT_F # Publish to NATS if is_publish is True if is_publish: - nats_conn = NATSConnection(nats_url) + nats_conn = NATSConnection(broker_url) nats_conn.connect() nats_conn.publish(subject, msg_json) nats_conn.close() @@ -571,18 +712,29 @@ def smartreceive(msg, fileserver_download_handler=_fetch_with_backoff, max_retri (base64 decoded payloads) and link transport (URL-based payloads). Args: - msg: NATS message to process (dict with payload data) + msg: NATS message to process (dict or JSON string with envelope data) fileserver_download_handler: Function to handle downloading data from file server URLs - max_retries: Maximum retry attempts for fetching URL - base_delay: Initial delay for exponential backoff in ms - max_delay: Maximum delay for exponential backoff in ms + Receives: (url, max_retries, base_delay, max_delay, correlation_id) + Returns: bytes (the downloaded data) + max_retries: Maximum retry attempts for fetching URL (default: 5) + base_delay: Initial delay for exponential backoff in ms (default: 100) + max_delay: Maximum delay for exponential backoff in ms (default: 5000) Returns: - dict: Envelope dictionary with metadata and 'payloads' field containing list of (dataname, data, type) tuples + dict: Envelope dictionary with metadata and 'payloads' field containing list of + (dataname, data, payload_type) tuples + + Example: + >>> env = smartreceive(msg) + >>> # env contains envelope metadata and payloads field + >>> # env["payloads"] = [(dataname1, data1, payload_type1), ...] + >>> for dataname, data, payload_type in env["payloads"]: + ... print("Received {} of type {}: {}".format(dataname, payload_type, data)) """ # Parse the JSON envelope json_data = msg if isinstance(msg, dict) else json.loads(msg) - log_trace(json_data.get("correlationId", ""), "Processing received message") + correlation_id = json_data.get("correlation_id", "") + log_trace(correlation_id, "Processing received message") # Process all payloads in the envelope payloads_list = [] @@ -596,43 +748,47 @@ def smartreceive(msg, fileserver_download_handler=_fetch_with_backoff, max_retri dataname = payload.get("dataname", "") if transport == "direct": - log_trace(json_data.get("correlationId", ""), + log_trace(correlation_id, "Direct transport - decoding payload '{}'".format(dataname)) # Extract base64 payload from the payload payload_b64 = payload.get("data", "") # Decode Base64 payload - import ubinascii - payload_bytes = ubinascii.a2b_base64(payload_b64.encode('utf-8')) + try: + import ubinascii + payload_bytes = ubinascii.a2b_base64(payload_b64.encode('utf-8')) + except ImportError: + import base64 + payload_bytes = base64.b64decode(payload_b64) # Deserialize based on type - data_type = payload.get("type", "") - data = _deserialize_data(payload_bytes, data_type, json_data.get("correlationId", "")) + payload_type = payload.get("payload_type", "") + data = _deserialize_data(payload_bytes, payload_type, correlation_id) - payloads_list.append((dataname, data, data_type)) + payloads_list.append((dataname, data, payload_type)) elif transport == "link": # Extract download URL from the payload url = payload.get("data", "") - log_trace(json_data.get("correlationId", ""), + log_trace(correlation_id, "Link transport - fetching '{}' from URL: {}".format(dataname, url)) # Fetch with exponential backoff downloaded_data = fileserver_download_handler( - url, max_retries, base_delay, max_delay, json_data.get("correlationId", "") + url, max_retries, base_delay, max_delay, correlation_id ) # Deserialize based on type - data_type = payload.get("type", "") - data = _deserialize_data(downloaded_data, data_type, json_data.get("correlationId", "")) + payload_type = payload.get("payload_type", "") + data = _deserialize_data(downloaded_data, payload_type, correlation_id) - payloads_list.append((dataname, data, data_type)) + payloads_list.append((dataname, data, payload_type)) else: raise ValueError("Unknown transport type for payload '{}': {}".format(dataname, transport)) - # Replace payloads field with the processed list of (dataname, data, type) tuples + # Replace payloads field with the processed list of (dataname, data, payload_type) tuples json_data["payloads"] = payloads_list return json_data @@ -651,11 +807,11 @@ def get_timestamp(): # Example usage if __name__ == "__main__": - print("NATSBridge for Micropython") - print("=========================") + print("NATSBridge - Bi-Directional Data Bridge") + print("=======================================") print("This module provides:") - print(" - MessageEnvelope: Message envelope structure") - print(" - MessagePayload: Payload structure") + print(" - MessageEnvelope: Message envelope structure with snake_case fields") + print(" - MessagePayload: Payload structure with payload_type field") print(" - smartsend: Send data via NATS with automatic transport selection") print(" - smartreceive: Receive and process messages from NATS") print(" - plik_oneshot_upload: Upload files to HTTP file server") @@ -663,10 +819,12 @@ if __name__ == "__main__": print() print("Usage:") print(" from nats_bridge import smartsend, smartreceive") - print(" data = [(\"message\", \"Hello World\", \"text\")]") - print(" env = smartsend(\"my.subject\", data)") + print() + print(" # Send data (list of (dataname, data, payload_type) tuples)") + print(" data = [(\"message\", \"Hello World!\", \"text\")]") + print(" env, env_json_str = smartsend(\"my.subject\", data)") print() print(" # On receiver:") - print(" payloads = smartreceive(msg)") - print(" for dataname, data, type in payloads:") - print(" print(f\"Received {dataname} of type {type}: {data}\")") \ No newline at end of file + print(" env = smartreceive(msg)") + print(" for dataname, data, payload_type in env[\"payloads\"]:") + print(" print(\"Received {} of type {}: {}\".format(dataname, payload_type, data))") \ No newline at end of file -- 2.49.1 From f8235e1a59b7fce7f0f9b2378f00e9663b7edc30 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 08:54:04 +0700 Subject: [PATCH 24/35] update --- examples/tutorial.md | 50 +++++++++++++++++++++-------------------- examples/walkthrough.md | 42 +++++++++++++++++----------------- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/examples/tutorial.md b/examples/tutorial.md index 9560f8a..6f58eab 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -109,11 +109,11 @@ from nats_bridge import smartsend # Send a text message (is_publish=True by default) data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") print("Message sent!") # Or use is_publish=False to get envelope and JSON without publishing -env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222", is_publish=False) +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=False) # env: MessageEnvelope object # env_json_str: JSON string for publishing to NATS ``` @@ -126,14 +126,14 @@ const { smartsend } = require('./src/NATSBridge'); // Send a text message (isPublish=true by default) await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } -], { natsUrl: "nats://localhost:4222" }); +], { brokerUrl: "nats://localhost:4222" }); console.log("Message sent!"); // Or use isPublish=false to get envelope and JSON without publishing const { env, env_json_str } = await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } -], { natsUrl: "nats://localhost:4222", isPublish: false }); +], { brokerUrl: "nats://localhost:4222", isPublish: false }); // env: MessageEnvelope object // env_json_str: JSON string for publishing to NATS ``` @@ -145,8 +145,8 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") -# env: msgEnvelope_v1 object with all metadata and payloads +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") +# env: msg_envelope_v1 object with all metadata and payloads # env_json_str: JSON string representation of the envelope for publishing println("Message sent!") ``` @@ -208,7 +208,7 @@ config = { # Send as dictionary type data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222") ``` #### JavaScript @@ -224,7 +224,7 @@ const config = { const { env, env_json_str } = await smartsend("/device/config", [ { dataname: "config", data: config, type: "dictionary" } -]); +], { brokerUrl: "nats://localhost:4222" }); ``` #### Julia @@ -239,7 +239,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data) +env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222") ``` ### Example 2: Sending Binary Data (Image) @@ -255,7 +255,7 @@ with open("image.png", "rb") as f: # Send as binary type data = [("user_image", image_data, "binary")] -env, env_json_str = smartsend("/chat/image", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222") ``` #### JavaScript @@ -269,7 +269,7 @@ const image_data = fs.readFileSync('image.png'); const { env, env_json_str } = await smartsend("/chat/image", [ { dataname: "user_image", data: image_data, type: "binary" } -]); +], { brokerUrl: "nats://localhost:4222" }); ``` #### Julia @@ -281,7 +281,7 @@ using NATSBridge image_data = read("image.png") data = [("user_image", image_data, "binary")] -env, env_json_str = smartsend("/chat/image", data) +env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222") ``` ### Example 3: Request-Response Pattern @@ -296,11 +296,11 @@ data = [("command", {"action": "read_sensor"}, "dictionary")] env, env_json_str = smartsend( "/device/command", data, - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", reply_to="/device/response", reply_to_msg_id="cmd-001" ) -# env: msgEnvelope_v1 object +# env: MessageEnvelope object # env_json_str: JSON string for publishing to NATS ``` @@ -361,7 +361,7 @@ large_data = os.urandom(2_000_000) # 2MB of random data env, env_json_str = smartsend( "/data/large", [("large_file", large_data, "binary")], - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", fileserver_url="http://localhost:8080", size_threshold=1_000_000 ) @@ -383,6 +383,7 @@ view.fill(42); // Fill with some data const { env, env_json_str } = await smartsend("/data/large", [ { dataname: "large_file", data: largeData, type: "binary" } ], { + brokerUrl: "nats://localhost:4222", fileserverUrl: "http://localhost:8080", sizeThreshold: 1_000_000 }); @@ -399,6 +400,7 @@ large_data = rand(UInt8, 2_000_000) env, env_json_str = smartsend( "/data/large", [("large_file", large_data, "binary")], + broker_url="nats://localhost:4222", fileserver_url="http://localhost:8080" ) @@ -425,7 +427,7 @@ data = [ ("user_avatar", image_data, "image") ] -env, env_json_str = smartsend("/chat/mixed", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222") ``` #### JavaScript @@ -446,7 +448,7 @@ const { env, env_json_str } = await smartsend("/chat/mixed", [ data: fs.readFileSync("avatar.png"), type: "image" } -]); +], { brokerUrl: "nats://localhost:4222" }); ``` #### Julia @@ -461,7 +463,7 @@ data = [ ("user_avatar", image_data, "image") ] -env, env_json_str = smartsend("/chat/mixed", data) +env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222") ``` ### Example 6: Table Data (Arrow IPC) @@ -483,7 +485,7 @@ df = pd.DataFrame({ # Send as table type data = [("students", df, "table")] -env, env_json_str = smartsend("/data/students", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222") ``` #### Julia @@ -500,7 +502,7 @@ df = DataFrame( ) data = [("students", df, "table")] -env, env_json_str = smartsend("/data/students", data) +env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222") ``` --- @@ -519,7 +521,7 @@ using NATSBridge # Send dictionary from Julia to JavaScript config = Dict("step_size" => 0.01, "iterations" => 1000) data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/analysis/config", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/analysis/config", data, broker_url="nats://localhost:4222") ``` #### JavaScript Receiver @@ -546,7 +548,7 @@ const { smartsend } = require('./src/NATSBridge'); const { env, env_json_str } = await smartsend("/data/transfer", [ { dataname: "message", data: "Hello from JS!", type: "text" } -]); +], { brokerUrl: "nats://localhost:4222" }); ``` #### Python Receiver @@ -568,7 +570,7 @@ for dataname, data, type in env["payloads"]: from nats_bridge import smartsend data = [("message", "Hello from Python!", "text")] -env, env_json_str = smartsend("/chat/python", data) +env, env_json_str = smartsend("/chat/python", data, broker_url="nats://localhost:4222") ``` #### Julia Receiver @@ -576,7 +578,7 @@ env, env_json_str = smartsend("/chat/python", data) ```julia using NATSBridge -env = smartreceive(msg, fileserverDownloadHandler) +env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff) for (dataname, data, type) in env["payloads"] if type == "text" println("Received from Python: $data") diff --git a/examples/walkthrough.md b/examples/walkthrough.md index b9f68fe..8f51475 100644 --- a/examples/walkthrough.md +++ b/examples/walkthrough.md @@ -136,6 +136,7 @@ class ChatUI { `/chat/${this.currentRoom}`, data, { + brokerUrl: window.config.broker_url, fileserverUrl: window.config.fileserver_url, sizeThreshold: window.config.size_threshold } @@ -288,8 +289,8 @@ Let's build a file transfer system that handles large files efficiently. const { smartsend } = require('./NATSBridge'); class FileUploadService { - constructor(natsUrl, fileserverUrl) { - this.natsUrl = natsUrl; + constructor(brokerUrl, fileserverUrl) { + this.brokerUrl = brokerUrl; this.fileserverUrl = fileserverUrl; } @@ -308,7 +309,7 @@ class FileUploadService { `/files/${recipient}`, data, { - natsUrl: this.natsUrl, + brokerUrl: this.brokerUrl, fileserverUrl: this.fileserverUrl, sizeThreshold: 1048576 } @@ -419,7 +420,7 @@ async function uploadFile(config) { const filePath = await rl.question('Enter file path: '); const recipient = await rl.question('Enter recipient: '); - const fileService = new FileUploadService(config.nats_url, config.fileserver_url); + const fileService = new FileUploadService(config.broker_url, config.fileserver_url); try { const env = await fileService.uploadFile(filePath, recipient); @@ -500,8 +501,8 @@ import time import random class SensorSender: - def __init__(self, nats_url: str, fileserver_url: str): - self.nats_url = nats_url + def __init__(self, broker_url: str, fileserver_url: str): + self.broker_url = broker_url self.fileserver_url = fileserver_url def send_reading(self, sensor_id: str, value: float, unit: str): @@ -518,7 +519,7 @@ class SensorSender: smartsend( f"/sensors/{sensor_id}", data, - nats_url=self.nats_url, + broker_url=self.broker_url, fileserver_url=self.fileserver_url ) @@ -537,7 +538,7 @@ class SensorSender: env, env_json_str = smartsend( f"/sensors/{sensor_id}/prepare", data, - nats_url=self.nats_url, + broker_url=self.broker_url, fileserver_url=self.fileserver_url, is_publish=False ) @@ -572,7 +573,7 @@ class SensorSender: smartsend( f"/sensors/batch", data, - nats_url=self.nats_url, + broker_url=self.broker_url, fileserver_url=self.fileserver_url ) else: @@ -631,17 +632,17 @@ Let's build an IoT device using Micropython that connects to NATS. import json class DeviceConfig: - def __init__(self, ssid, password, nats_url, device_id): + def __init__(self, ssid, password, broker_url, device_id): self.ssid = ssid self.password = password - self.nats_url = nats_url + self.broker_url = broker_url self.device_id = device_id def to_dict(self): return { "ssid": self.ssid, "password": self.password, - "nats_url": self.nats_url, + "broker_url": self.broker_url, "device_id": self.device_id } ``` @@ -656,7 +657,7 @@ import json class DeviceBridge: def __init__(self, config): self.config = config - self.nats_url = config.nats_url + self.broker_url = config.broker_url def connect(self): # Connect to WiFi @@ -676,7 +677,7 @@ class DeviceBridge: smartsend( f"/devices/{self.config.device_id}/status", data, - nats_url=self.nats_url + broker_url=self.broker_url ) def send_sensor_data(self, sensor_id, value, unit): @@ -687,7 +688,7 @@ class DeviceBridge: smartsend( f"/devices/{self.config.device_id}/sensors/{sensor_id}", data, - nats_url=self.nats_url + broker_url=self.broker_url ) def receive_commands(self, callback): @@ -725,7 +726,7 @@ import random config = DeviceConfig( ssid="MyNetwork", password="password123", - nats_url="nats://localhost:4222", + broker_url="nats://localhost:4222", device_id="device-001" ) @@ -774,8 +775,8 @@ import pyarrow as pa import io class DashboardServer: - def __init__(self, nats_url, fileserver_url): - self.nats_url = nats_url + def __init__(self, broker_url, fileserver_url): + self.broker_url = broker_url self.fileserver_url = fileserver_url def broadcast_data(self, df): @@ -792,7 +793,7 @@ class DashboardServer: smartsend( "/dashboard/data", data, - nats_url=self.nats_url, + broker_url=self.broker_url, fileserver_url=self.fileserver_url ) @@ -836,6 +837,7 @@ class DashboardUI { const { env, env_json_str } = await smartsend("/dashboard/request", [ { dataname: "request", data: { type: "refresh" }, type: "dictionary" } ], { + brokerUrl: window.config.broker_url, fileserverUrl: window.config.fileserver_url }); } @@ -954,7 +956,7 @@ def send_batch_readings(self, readings): smartsend( "/sensors/batch", [("batch", arrow_data, "table")], - nats_url=self.nats_url + broker_url=self.broker_url ) ``` -- 2.49.1 From d94761c866a7bb329556363493a3ff2ef2d7516e Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 09:44:08 +0700 Subject: [PATCH 25/35] update --- src/NATSBridge.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index 03044fe..eda6441 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -431,7 +431,8 @@ function smartsend( receiver_id::String = "", reply_to::String = "", reply_to_msg_id::String = "", - is_publish::Bool = true # some time the user want to get env and env_json_str from this function without publishing the msg + is_publish::Bool = true, # some time the user want to get env and env_json_str from this function without publishing the msg + NATS_connection::Union{NATS.Connection, Nothing} = nothing # a provided connection saves establishing connection overhead. ) where {T1<:Any} # Generate correlation ID if not provided @@ -515,8 +516,12 @@ function smartsend( ) env_json_str = envelope_to_json(env) # Convert envelope to JSON - if is_publish + if is_publish == false + # skip publish a message + elseif is_publish == true && NATS_connection === nothing publish_message(broker_url, subject, env_json_str, cid) # Publish message to NATS + elseif is_publish == true && NATS_connection !== nothing + publish_message(NATS_connection, subject, env_json_str, cid) # Publish message to NATS end return (env, env_json_str) @@ -673,6 +678,10 @@ connection management and logging. """ function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) conn = NATS.connect(broker_url) # Create NATS connection + publish_message(conn, subject, message, correlation_id) +end + +function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) try NATS.publish(conn, subject, message) # Publish message to NATS log_trace(correlation_id, "Message published to $subject") # Log successful publish @@ -1055,7 +1064,7 @@ function plik_oneshot_upload(file_server_url::String, filepath::String) end function _get_payload_bytes(data) - @error "didn't implement yet" + @error "Didn't implement yet. The developer will implement this function later." end -- 2.49.1 From df0bbc7327ddf2e9ca0c20321d70bb1a4659a131 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 09:58:10 +0700 Subject: [PATCH 26/35] update --- src/NATSBridge.jl | 53 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index eda6441..c2acaa0 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -380,9 +380,10 @@ Each payload can have a different type, enabling mixed-content messages (e.g., c - `sender_name::String = "NATSBridge"` - Name of the sender - `receiver_name::String = ""` - Name of the receiver (empty string means broadcast) - `receiver_id::String = ""` - UUID of the receiver (empty string means broadcast) - - `reply_to::String = ""` - Topic to reply to (empty string if no reply expected) - - `reply_to_msg_id::String = ""` - Message ID this message is replying to - - `is_publish::Bool = true` - Whether to automatically publish the message to NATS + - `reply_to::String = ""` - Topic to reply to (empty string if no reply expected) + - `reply_to_msg_id::String = ""` - Message ID this message is replying to + - `is_publish::Bool = true` - Whether to automatically publish the message to NATS + - `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection (if provided, uses this connection instead of creating a new one; saves connection establishment overhead) # Return: - A tuple `(env, env_json_str)` where: @@ -653,7 +654,7 @@ end """ publish_message - Publish message to NATS -This internal function publishes a message to a NATS subject with proper +This function publishes a message to a NATS subject with proper connection management and logging. # Arguments: @@ -666,21 +667,51 @@ connection management and logging. - `nothing` - This function performs publishing but returns nothing # Example - ```jldoctest - using NATS +```jldoctest +using NATS - # Prepare JSON message - message = "{\"correlation_id\":\"abc123\",\"payload\":\"test\"}" +# Prepare JSON message +message = "{\"correlation_id\":\"abc123\",\"payload\":\"test\"}" - # Publish to NATS - publish_message("nats://localhost:4222", "my.subject", message, "abc123") - ``` +# Publish to NATS +publish_message("nats://localhost:4222", "my.subject", message, "abc123") +``` """ function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) conn = NATS.connect(broker_url) # Create NATS connection publish_message(conn, subject, message, correlation_id) end +""" publish_message - Publish message to NATS using pre-existing connection +This function publishes a message to a NATS subject using a pre-existing NATS connection, +avoiding the overhead of connection establishment. + +# Arguments: + - `conn::NATS.Connection` - Pre-existing NATS connection + - `subject::String` - NATS subject to publish to (e.g., "/agent/wine/api/v1/prompt") + - `message::String` - JSON message to publish + - `correlation_id::String` - Correlation ID for tracing and logging + +# Return: + - `nothing` - This function performs publishing but returns nothing + +# Example +```jldoctest +using NATS + +# Prepare JSON message +message = "{\"correlation_id\":\"abc123\",\"payload\":\"test\"}" + +# Create connection once and reuse for multiple publishes +conn = NATS.connect("nats://localhost:4222") +publish_message(conn, "my.subject", message, "abc123") +# Connection is automatically drained after publish +``` + +# Use Case: +Use this version when you already have an established NATS connection and want to publish +multiple messages without the overhead of creating a new connection for each publish. +""" function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) try NATS.publish(conn, subject, message) # Publish message to NATS -- 2.49.1 From ae2de5fc62d24f7a320e134743cfdfecfbdfd05c Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 10:33:30 +0700 Subject: [PATCH 27/35] update --- AI_prompt.txt | 18 +++++- docs/architecture.md | 64 +++++++++++++++++++- docs/implementation.md | 129 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) diff --git a/AI_prompt.txt b/AI_prompt.txt index b6ad565..3595c6c 100644 --- a/AI_prompt.txt +++ b/AI_prompt.txt @@ -14,6 +14,22 @@ Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement Create a walkthrough for Julia service-A service sending a mix-content chat message to Julia service-B. the chat message must includes -I update architecture.md and NATSBridge.jl. Use them as ground truth and update implementation.md accordingly. Also look for any inconsistency. + + + + + +I updated the following: +- NATSBridge.jl. Essentially I add NATS_connection keyword and new publish_message function to support the keyword. + +Use them and ONLY them as ground truth. + +Then update the following files accordingly: +- architecture.md +- implementation.md + +All API should be semantically consistent and naming should be consistent across the board. + + diff --git a/docs/architecture.md b/docs/architecture.md index 09dac03..d89d0e0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -395,10 +395,25 @@ function smartsend( receiver_id::String = "", reply_to::String = "", reply_to_msg_id::String = "", - is_publish::Bool = true # Whether to automatically publish to NATS + is_publish::Bool = true, # Whether to automatically publish to NATS + NATS_connection::Union{NATS.Connection, Nothing} = nothing # Pre-existing NATS connection (optional, saves connection overhead) ) ``` +**New Keyword Parameter:** +- `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios where connection reuse provides performance benefits. + +**Connection Handling Logic:** +```julia +if is_publish == false + # skip publish a message +elseif is_publish == true && NATS_connection === nothing + publish_message(broker_url, subject, env_json_str, cid) # Creates new connection +elseif is_publish == true && NATS_connection !== nothing + publish_message(NATS_connection, subject, env_json_str, cid) # Uses provided connection +end +``` + **Return Value:** - Returns a tuple `(env, env_json_str)` where: - `env::msg_envelope_v1` - The envelope object containing all metadata and payloads @@ -459,6 +474,53 @@ end **Note:** The `fileserver_download_handler` receives `(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)` and returns `Vector{UInt8}`. +#### publish_message Function + +The `publish_message` function provides two overloads for publishing messages to NATS: + +**Overload 1 - URL-based publishing (creates new connection):** +```julia +function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) + conn = NATS.connect(broker_url) # Create NATS connection + publish_message(conn, subject, message, correlation_id) +end +``` + +**Overload 2 - Connection-based publishing (uses pre-existing connection):** +```julia +function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) + try + NATS.publish(conn, subject, message) # Publish message to NATS + log_trace(correlation_id, "Message published to $subject") # Log successful publish + finally + NATS.drain(conn) # Ensure connection is closed properly + end +end +``` + +**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. + +**Integration with smartsend:** +```julia +# When NATS_connection is provided to smartsend, it uses the connection-based publish_message +env, env_json_str = smartsend( + "my.subject", + [("data", payload_data, "type")], + NATS_connection=my_connection, # Pre-existing connection + is_publish=true +) +# Uses: publish_message(NATS_connection, subject, env_json_str, cid) + +# When NATS_connection is not provided, it uses the URL-based publish_message +env, env_json_str = smartsend( + "my.subject", + [("data", payload_data, "type")], + broker_url="nats://localhost:4222", + is_publish=true +) +# Uses: publish_message(broker_url, subject, env_json_str, cid) +``` + ### JavaScript Implementation #### Dependencies diff --git a/docs/implementation.md b/docs/implementation.md index 5792e3b..07727e0 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -337,6 +337,30 @@ env, env_json_str = smartsend( # env_json_str: JSON string for publishing ``` +**Julia (Sender/Receiver) with NATS_connection for connection reuse:** +```julia +using NATSBridge + +# Create connection once for high-frequency publishing +conn = NATS.connect("nats://localhost:4222") + +# Send multiple messages using the same connection (saves connection overhead) +for i in 1:100 + config = Dict("iteration" => i, "data" => rand()) + smartsend( + "control", + [("config", config, "dictionary")], + NATS_connection=conn, # Reuse connection + is_publish=true + ) +end + +# Close connection when done +NATS.close(conn) +``` + +**Use Case:** High-frequency publishing scenarios where connection reuse provides performance benefits by avoiding the overhead of establishing a new NATS connection for each message. + **JavaScript (Sender/Receiver):** ```javascript const { smartsend } = require('./src/NATSBridge'); @@ -478,6 +502,111 @@ env, env_json_str = smartsend( # env_json_str: JSON string for publishing ``` +#### smartsend Function Signature (Julia) + +```julia +function smartsend( + subject::String, + data::AbstractArray{Tuple{String, Any, String}, 1}; # List of (dataname, data, type) tuples + broker_url::String = DEFAULT_BROKER_URL, # NATS server URL + fileserver_url = DEFAULT_FILESERVER_URL, + fileserver_upload_handler::Function = plik_oneshot_upload, + size_threshold::Int = DEFAULT_SIZE_THRESHOLD, + correlation_id::Union{String, Nothing} = nothing, + msg_purpose::String = "chat", + sender_name::String = "NATSBridge", + receiver_name::String = "", + receiver_id::String = "", + reply_to::String = "", + reply_to_msg_id::String = "", + is_publish::Bool = true, + NATS_connection::Union{NATS.Connection, Nothing} = nothing # Pre-existing NATS connection (optional) +) +``` + +**New Keyword Parameter:** +- `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios. + +**Connection Handling Logic:** +```julia +if is_publish == false + # skip publish +elseif is_publish == true && NATS_connection === nothing + publish_message(broker_url, subject, env_json_str, cid) # Creates new connection +elseif is_publish == true && NATS_connection !== nothing + publish_message(NATS_connection, subject, env_json_str, cid) # Uses provided connection +end +``` + +**Example with pre-existing connection:** +```julia +using NATSBridge + +# Create connection once +conn = NATS.connect("nats://localhost:4222") + +# Send multiple messages using the same connection +for i in 1:100 + data = rand(1000) + smartsend( + "analysis_results", + [("table_data", data, "table")], + NATS_connection=conn, # Reuse connection + is_publish=true + ) +end + +# Close connection when done +NATS.close(conn) +``` + +#### publish_message Function + +The `publish_message` function provides two overloads for publishing messages to NATS: + +**Overload 1 - URL-based publishing (creates new connection):** +```julia +function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) + conn = NATS.connect(broker_url) # Create NATS connection + publish_message(conn, subject, message, correlation_id) +end +``` + +**Overload 2 - Connection-based publishing (uses pre-existing connection):** +```julia +function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) + try + NATS.publish(conn, subject, message) # Publish message to NATS + log_trace(correlation_id, "Message published to $subject") + finally + NATS.drain(conn) # Ensure connection is closed properly + end +end +``` + +**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. + +**Integration with smartsend:** +```julia +# When NATS_connection is provided to smartsend, it uses the connection-based publish_message +env, env_json_str = smartsend( + "my.subject", + [("data", payload_data, "type")], + NATS_connection=my_connection, # Pre-existing connection + is_publish=true +) +# Uses: publish_message(NATS_connection, subject, env_json_str, cid) + +# When NATS_connection is not provided, it uses the URL-based publish_message +env, env_json_str = smartsend( + "my.subject", + [("data", payload_data, "type")], + broker_url="nats://localhost:4222", + is_publish=true +) +# Uses: publish_message(broker_url, subject, env_json_str, cid) +``` + #### JavaScript (Receiver) ```javascript const { smartreceive } = require('./src/NATSBridge'); -- 2.49.1 From 6a862ef24305d4fd6851b166aa93c1ee8bf55cf8 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 12:09:00 +0700 Subject: [PATCH 28/35] update --- docs/architecture.md | 52 ++++++++++++++++++++++--- docs/implementation.md | 87 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 8 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d89d0e0..3275141 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -369,6 +369,25 @@ graph TD ## Implementation Details +### API Consistency Across Languages + +**High-Level API (Consistent Across All Languages):** +- `smartsend(subject, data, ...)` - Main publishing function +- `smartreceive(msg, ...)` - Main receiving function +- Message envelope structure (`msg_envelope_v1` / `MessageEnvelope`) +- Payload structure (`msg_payload_v1` / `MessagePayload`) +- Transport strategy (direct vs link based on size threshold) +- Supported payload types: text, dictionary, table, image, audio, video, binary + +**Low-Level Native Functions (Language-Specific Conventions):** +- Julia: `NATS.connect()`, `publish_message()`, function overloading +- JavaScript: `nats.js` client, native async/await patterns +- Python: `nats-python` client, native async/await patterns + +**Connection Reuse Pattern:** +- **Julia:** Uses `NATS_connection` keyword parameter with function overloading +- **JavaScript/Python:** Achieved by creating NATS client outside the function and reusing it in custom handlers + ### Julia Implementation #### Dependencies @@ -400,7 +419,7 @@ function smartsend( ) ``` -**New Keyword Parameter:** +**Keyword Parameter - NATS_connection:** - `NATS_connection::Union{NATS.Connection, Nothing} = nothing` - Pre-existing NATS connection. When provided, `smartsend` uses this connection instead of creating a new one, avoiding the overhead of connection establishment. This is useful for high-frequency publishing scenarios where connection reuse provides performance benefits. **Connection Handling Logic:** @@ -498,7 +517,7 @@ function publish_message(conn::NATS.Connection, subject::String, message::String end ``` -**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. +**Use Case:** Use the connection-based overload when you already have an established NATS connection and want to publish multiple messages without the overhead of creating a new connection for each publish. This is a Julia-specific optimization that leverages function overloading. **Integration with smartsend:** ```julia @@ -521,6 +540,10 @@ env, env_json_str = smartsend( # Uses: publish_message(broker_url, subject, env_json_str, cid) ``` +**API Consistency Note:** +- **High-level API (smartsend, smartreceive):** Uses consistent naming across all three languages (Julia, JavaScript, Python/Micropython) +- **Low-level native functions (NATS.connect(), publish_message()):** Follow the conventions of the specific language ecosystem and do not require cross-language consistency + ### JavaScript Implementation #### Dependencies @@ -551,8 +574,11 @@ async function smartsend( - `receiver_id` (String) - Message receiver ID (default: `""`) - `reply_to` (String) - Topic to reply to (default: `""`) - `reply_to_msg_id` (String) - Message ID this message is replying to (default: `""`) +- `is_publish` (Boolean) - Whether to automatically publish the message to NATS (default: `true`) - `fileserver_upload_handler` (Function) - Custom upload handler function +**Note:** JavaScript uses `is_publish` option (instead of `NATS_connection` keyword) to control automatic publishing behavior. Connection reuse can be achieved by creating a NATS client outside the function and reusing it in a custom `fileserver_upload_handler` or custom publish implementation. + **Return Value:** - Returns a Promise that resolves to an object containing: - `env` - The envelope object containing all metadata and payloads @@ -618,26 +644,40 @@ async function smartreceive(msg, options = {}) #### smartsend Function ```python -async def smartsend( +def smartsend( subject: str, data: List[Tuple[str, Any, str]], # List of (dataname, data, type) tuples - options: Dict = {} -) + broker_url: str = DEFAULT_BROKER_URL, + fileserver_url: str = DEFAULT_FILESERVER_URL, + fileserver_upload_handler: Callable = plik_oneshot_upload, + size_threshold: int = DEFAULT_SIZE_THRESHOLD, + correlation_id: Union[str, None] = None, + msg_purpose: str = "chat", + sender_name: str = "NATSBridge", + receiver_name: str = "", + receiver_id: str = "", + reply_to: str = "", + reply_to_msg_id: str = "", + is_publish: bool = True +) -> Tuple[MessageEnvelope, str] ``` **Options:** - `broker_url` (str) - NATS server URL (default: `"nats://localhost:4222"`) - `fileserver_url` (str) - Base URL of the file server (default: `"http://localhost:8080"`) - `size_threshold` (int) - Threshold in bytes for transport selection (default: `1048576` = 1MB) -- `correlation_id` (str) - Optional correlation ID for tracing +- `correlation_id` (str) - Optional correlation ID for tracing (auto-generated if None) - `msg_purpose` (str) - Purpose of the message (default: `"chat"`) - `sender_name` (str) - Sender name (default: `"NATSBridge"`) - `receiver_name` (str) - Message receiver name (default: `""`) - `receiver_id` (str) - Message receiver ID (default: `""`) - `reply_to` (str) - Topic to reply to (default: `""`) - `reply_to_msg_id` (str) - Message ID this message is replying to (default: `""`) +- `is_publish` (bool) - Whether to automatically publish the message to NATS (default: `True`) - `fileserver_upload_handler` (Callable) - Custom upload handler function +**Note:** Python uses `is_publish` parameter (instead of `NATS_connection` keyword) to control automatic publishing behavior. Connection reuse can be achieved by creating a NATS client outside the function and reusing it in a custom `fileserver_upload_handler` or custom publish implementation. + **Return Value:** - Returns a tuple `(env, env_json_str)` where: - `env` - The envelope dictionary containing all metadata and payloads diff --git a/docs/implementation.md b/docs/implementation.md index 07727e0..8da6ce3 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -316,6 +316,30 @@ julia test/scenario3_julia_to_julia.jl node test/scenario3_julia_to_julia.js ``` +## API Consistency Across Languages + +**High-Level API (Consistent Across All Languages):** +- `smartsend(subject, data, ...)` - Main publishing function +- `smartreceive(msg, ...)` - Main receiving function +- Message envelope structure (`msg_envelope_v1` / `MessageEnvelope`) +- Payload structure (`msg_payload_v1` / `MessagePayload`) +- Transport strategy (direct vs link based on size threshold) +- Supported payload types: text, dictionary, table, image, audio, video, binary + +**Low-Level Native Functions (Language-Specific Conventions):** +- Julia: `NATS.connect()`, `publish_message()`, function overloading +- JavaScript: `nats.js` client, native async/await patterns +- Python: `nats-python` client, native async/await patterns + +**Connection Reuse Pattern - Key Differences:** +- **Julia:** Uses `NATS_connection` keyword parameter with function overloading for automatic connection management +- **JavaScript/Python:** Achieved by creating NATS client outside the function and reusing it in custom handlers or custom publish implementations + +**Why the Difference?** +- Julia supports function overloading and keyword arguments, allowing `NATS_connection` to be passed as an optional parameter +- JavaScript/Python use a simpler `is_publish` option to control automatic publishing +- For connection reuse in JavaScript/Python, create a NATS client once and reuse it in your custom `fileserver_upload_handler` or custom publish logic + ## Usage ### Scenario 1: Command & Control (Small Dictionary) @@ -373,9 +397,39 @@ const config = { threshold: 0.5 }; +// Use is_publish option to control automatic publishing await smartsend("control", [ { dataname: "config", data: config, type: "dictionary" } -]); +], { + is_publish: true // Automatically publish to NATS +}); +``` + +**Connection Reuse in JavaScript:** +To achieve connection reuse in JavaScript, create a NATS client outside the function and use it in a custom `fileserver_upload_handler` or custom publish implementation: + +```javascript +const { connect } = require('nats'); +const { smartsend } = require('./src/NATSBridge'); + +// Create connection once +const nc = await connect({ servers: ['nats://localhost:4222'] }); + +// Send multiple messages using the same connection +for (let i = 0; i < 100; i++) { + const config = { iteration: i, data: Math.random() }; + + // Option 1: Use is_publish=false and publish manually with your connection + const { env, env_json_str } = await smartsend("control", [ + { dataname: "config", data: config, type: "dictionary" } + ], { is_publish: false }); + + // Publish with your existing connection + await nc.publish("control", env_json_str); +} + +// Close connection when done +await nc.close(); ``` **Python/Micropython (Sender/Receiver):** @@ -390,7 +444,32 @@ config = { "threshold": 0.5 } -smartsend("control", [("config", config, "dictionary")]) +# Use is_publish parameter to control automatic publishing +smartsend("control", [("config", config, "dictionary")], is_publish=True) +``` + +**Connection Reuse in Python:** +To achieve connection reuse in Python, create a NATS client outside the function and use it in a custom `fileserver_upload_handler` or custom publish implementation: + +```python +from nats_bridge import smartsend +import nats + +# Create connection once +nc = await nats.connect("nats://localhost:4222") + +# Send multiple messages using the same connection +for i in range(100): + config = {"iteration": i, "data": random.random()} + + # Option 1: Use is_publish=False and publish manually with your connection + env, env_json_str = smartsend("control", [("config", config, "dictionary")], is_publish=False) + + # Publish with your existing connection + await nc.publish("control", env_json_str) + +# Close connection when done +await nc.close() ``` ### Basic Multi-Payload Example @@ -607,6 +686,10 @@ env, env_json_str = smartsend( # Uses: publish_message(broker_url, subject, env_json_str, cid) ``` +**API Consistency Note:** +- **Julia:** Uses `NATS_connection` keyword parameter with function overloading for automatic connection management +- **JavaScript/Python:** Use `is_publish` option and achieve connection reuse by creating NATS client outside the function and reusing it in custom handlers or custom publish implementations + #### JavaScript (Receiver) ```javascript const { smartreceive } = require('./src/NATSBridge'); -- 2.49.1 From be94c6276017b101ad47a6ec801166e02c4ea01e Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 12:24:02 +0700 Subject: [PATCH 29/35] update --- README.md | 87 ++++++++++++++++++++++++++++++++++++++----------------- etc.jl | 9 ++++++ 2 files changed, 69 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 6b81012..bca52f7 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ from nats_bridge import smartsend # Send a text message data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data, nats_url="nats://localhost:4222") +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") print("Message sent!") ``` @@ -185,7 +185,7 @@ const { smartsend } = require('./src/NATSBridge'); // Send a text message const { env, env_json_str } = await smartsend("/chat/room1", [ { dataname: "message", data: "Hello World", type: "text" } -], { natsUrl: "nats://localhost:4222" }); +], { broker_url: "nats://localhost:4222" }); console.log("Message sent!"); ``` @@ -197,7 +197,7 @@ using NATSBridge # Send a text message data = [("message", "Hello World", "text")] -env, env_json_str = NATSBridge.smartsend("/chat/room1", data; nats_url="nats://localhost:4222") +env, env_json_str = NATSBridge.smartsend("/chat/room1", data; broker_url="nats://localhost:4222") println("Message sent!") ``` @@ -313,7 +313,7 @@ from nats_bridge import smartsend env, env_json_str = smartsend( subject, # NATS subject to publish to data, # List of (dataname, data, type) tuples - nats_url="nats://localhost:4222", # NATS server URL + broker_url="nats://localhost:4222", # NATS server URL fileserver_url="http://localhost:8080", # File server URL fileserver_upload_handler=plik_oneshot_upload, # Upload handler function size_threshold=1_000_000, # Threshold in bytes (default: 1MB) @@ -337,18 +337,18 @@ const { env, env_json_str } = await smartsend( subject, // NATS subject data, // Array of {dataname, data, type} { - natsUrl: "nats://localhost:4222", - fileserverUrl: "http://localhost:8080", - fileserverUploadHandler: customUploadHandler, - sizeThreshold: 1_000_000, - correlationId: "custom-id", - msgPurpose: "chat", - senderName: "NATSBridge", - receiverName: "", - receiverId: "", - replyTo: "", - replyToMsgId: "", - isPublish: true // Whether to automatically publish to NATS + broker_url: "nats://localhost:4222", + fileserver_url: "http://localhost:8080", + fileserver_upload_handler: customUploadHandler, + size_threshold: 1_000_000, + correlation_id: "custom-id", + msg_purpose: "chat", + sender_name: "NATSBridge", + receiver_name: "", + receiver_id: "", + reply_to: "", + reply_to_msg_id: "", + is_publish: true // Whether to automatically publish to NATS } ); ``` @@ -361,9 +361,9 @@ using NATSBridge env, env_json_str = NATSBridge.smartsend( subject, # NATS subject data::AbstractArray{Tuple{String, Any, String}}; # List of (dataname, data, type) - nats_url::String = "nats://localhost:4222", + broker_url::String = "nats://localhost:4222", fileserver_url = "http://localhost:8080", - fileserverUploadHandler::Function = plik_oneshot_upload, + fileserver_upload_handler::Function = plik_oneshot_upload, size_threshold::Int = 1_000_000, correlation_id::Union{String, Nothing} = nothing, msg_purpose::String = "chat", @@ -372,7 +372,8 @@ env, env_json_str = NATSBridge.smartsend( receiver_id::String = "", reply_to::String = "", reply_to_msg_id::String = "", - is_publish::Bool = true # Whether to automatically publish to NATS + is_publish::Bool = true, # Whether to automatically publish to NATS + NATS_connection::Union{NATS.Connection, Nothing} = nothing # Pre-existing NATS connection (optional, saves connection overhead) ) # Returns: (msgEnvelope_v1, JSON string) # - env: msgEnvelope_v1 object with all envelope metadata and payloads @@ -423,9 +424,9 @@ const env = await smartreceive( using NATSBridge # Note: msg is a NATS.Msg object passed from the subscription callback -env, env_json_str = NATSBridge.smartreceive( +env = NATSBridge.smartreceive( msg::NATS.Msg; - fileserverDownloadHandler::Function = _fetch_with_backoff, + fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, base_delay::Int = 100, max_delay::Int = 5000 @@ -433,6 +434,35 @@ env, env_json_str = NATSBridge.smartreceive( # Returns: Dict with envelope metadata and payloads array ``` +### publish_message + +Publish a message to a NATS subject. This function is available in Julia with two overloads: + +#### Julia + +**Using broker URL (creates new connection):** +```julia +using NATSBridge, NATS + +# Publish with URL - creates a new connection +NATSBridge.publish_message( + "nats://localhost:4222", # broker_url + "/chat/room1", # subject + "{\"correlation_id\":\"abc123\"}", # message + "abc123" # correlation_id +) +``` + +**Using pre-existing connection (saves connection overhead):** +```julia +using NATSBridge, NATS + +# Create connection once and reuse +conn = NATS.connect("nats://localhost:4222") +NATSBridge.publish_message(conn, "/chat/room1", "{\"correlation_id\":\"abc123\"}", "abc123") +# Connection is automatically drained after publish +``` + --- ## Payload Types @@ -488,7 +518,7 @@ smartsend("/topic", data, fileserver_url="http://localhost:8080") ```javascript await smartsend("/topic", [ { dataname: "file", data: largeData, type: "binary" } -], { fileserverUrl: "http://localhost:8080" }); +], { fileserver_url: "http://localhost:8080" }); ``` #### Julia @@ -529,7 +559,7 @@ const { env, env_json_str } = await smartsend("/chat/room1", [ { dataname: "user_avatar", data: image_data, type: "image" }, { dataname: "large_document", data: large_file_data, type: "binary" } ], { - fileserverUrl: "http://localhost:8080" + fileserver_url: "http://localhost:8080" }); ``` @@ -653,9 +683,10 @@ from nats_bridge import smartsend env, env_json_str = smartsend( "/device/command", [("command", {"action": "read_sensor"}, "dictionary")], + broker_url="nats://localhost:4222", reply_to="/device/response" ) -# env: msgEnvelope_v1 object +# env: MessageEnvelope object # env_json_str: JSON string for publishing to NATS # The env_json_str can also be published directly using NATS request-reply pattern @@ -697,7 +728,8 @@ const { smartsend } = require('./src/NATSBridge'); const { env, env_json_str } = await smartsend("/device/command", [ { dataname: "command", data: { action: "read_sensor" }, type: "dictionary" } ], { - replyTo: "/device/response" + broker_url: "nats://localhost:4222", + reply_to: "/device/response" }); ``` @@ -739,6 +771,7 @@ using NATSBridge env, env_json_str = NATSBridge.smartsend( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; + broker_url="nats://localhost:4222", reply_to="/device/response" ) ``` @@ -755,7 +788,7 @@ const NATS_URL = "nats://localhost:4222" function test_responder() conn = NATS.connect(NATS_URL) NATS.subscribe(conn, SUBJECT) do msg - env, env_json_str = NATSBridge.smartreceive(msg, fileserverDownloadHandler) + env = NATSBridge.smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) for (dataname, data, type) in env["payloads"] if dataname == "command" && data["action"] == "read_sensor" response = Dict("sensor_id" => "sensor-001", "value" => 42.5) @@ -790,7 +823,7 @@ async def main(): # Send sensor data data = [("temperature", "25.5", "text"), ("humidity", 65, "dictionary")] - smartsend("/device/sensors", data, nats_url="nats://localhost:4222") + smartsend("/device/sensors", data, broker_url="nats://localhost:4222") # Receive commands - msg comes from the callback async def message_handler(msg): diff --git a/etc.jl b/etc.jl index e69de29..878e279 100644 --- a/etc.jl +++ b/etc.jl @@ -0,0 +1,9 @@ +Task: Update README.md to reflect recent changes in NATSbridge package. + +Context: the package has been updated with the NATS_connection keyword and the publish_message function. + +Requirements: + +Source of Truth: Treat the updated NATSbridge code as the definitive source. Update README.md to align exactly with these changes. +API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. +Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. \ No newline at end of file -- 2.49.1 From 1299febcdca50a17f5fbf7f979d3e29d7ce8162a Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 14:25:08 +0700 Subject: [PATCH 30/35] update --- README.md | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bca52f7..7e0fc89 100644 --- a/README.md +++ b/README.md @@ -701,18 +701,24 @@ from nats_bridge import smartreceive, smartsend # Configuration SUBJECT = "/device/command" -REPLY_SUBJECT = "/device/response" NATS_URL = "nats://localhost:4222" async def main(): nc = await nats.connect(NATS_URL) async def message_handler(msg): + # Receive and parse the incoming message envelope env = smartreceive(msg.data) + + # Extract reply_to from the envelope metadata + reply_to = env["reply_to"] + for dataname, data, type in env["payloads"]: if data.get("action") == "read_sensor": response = {"sensor_id": "sensor-001", "value": 42.5} - smartsend(REPLY_SUBJECT, [("data", response, "dictionary")]) + # Send response to the reply_to subject from the request + if reply_to: + smartsend(reply_to, [("data", response, "dictionary")]) sid = await nc.subscribe(SUBJECT, cb=message_handler) await asyncio.sleep(120) @@ -740,7 +746,6 @@ const { connect } = require('nats'); // Configuration const SUBJECT = "/device/command"; -const REPLY_SUBJECT = "/device/response"; const NATS_URL = "nats://localhost:4222"; async function main() { @@ -750,12 +755,19 @@ async function main() { for await (const msg of sub) { const env = await smartreceive(msg); + + // Extract reply_to from the envelope metadata + const replyTo = env["reply_to"]; + for (const payload of env.payloads) { if (payload.dataname === "command" && payload.data.action === "read_sensor") { const response = { sensor_id: "sensor-001", value: 42.5 }; - await smartsend(REPLY_SUBJECT, [ - { dataname: "data", data: response, type: "dictionary" } - ]); + // Send response to the reply_to subject from the request + if (replyTo) { + await smartsend(replyTo, [ + { dataname: "data", data: response, type: "dictionary" } + ]); + } } } } @@ -782,17 +794,23 @@ using NATS, NATSBridge # Configuration const SUBJECT = "/device/command" -const REPLY_SUBJECT = "/device/response" const NATS_URL = "nats://localhost:4222" function test_responder() conn = NATS.connect(NATS_URL) NATS.subscribe(conn, SUBJECT) do msg env = NATSBridge.smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) + + # Extract reply_to from the envelope metadata + reply_to = env["reply_to"] + for (dataname, data, type) in env["payloads"] if dataname == "command" && data["action"] == "read_sensor" response = Dict("sensor_id" => "sensor-001", "value" => 42.5) - smartsend(REPLY_SUBJECT, [("data", response, "dictionary")]) + # Send response to the reply_to subject from the request + if !isempty(reply_to) + smartsend(reply_to, [("data", response, "dictionary")]) + end end end end -- 2.49.1 From 3e1c8d563e490d63bfb2e6f402c6fd07f1e48121 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 15:20:29 +0700 Subject: [PATCH 31/35] update --- AI_prompt.txt | 18 ++++++++ docs/architecture.md | 6 +++ docs/implementation.md | 1 + src/NATSBridge.js | 101 +++++++++++------------------------------ 4 files changed, 51 insertions(+), 75 deletions(-) diff --git a/AI_prompt.txt b/AI_prompt.txt index 3595c6c..5120027 100644 --- a/AI_prompt.txt +++ b/AI_prompt.txt @@ -33,3 +33,21 @@ All API should be semantically consistent and naming should be consistent across + + + + + +Task: Update NATSBridge.js to reflect recent changes in NATSBridge.jl and docs + +Context: NATSBridge.jl and docs has been updated. + +Requirements: + +Source of Truth: Treat the updated NATSBridge.jl and docs as the definitive source. +API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. +Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. + + + + diff --git a/docs/architecture.md b/docs/architecture.md index 3275141..4a5f29b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -388,6 +388,12 @@ graph TD - **Julia:** Uses `NATS_connection` keyword parameter with function overloading - **JavaScript/Python:** Achieved by creating NATS client outside the function and reusing it in custom handlers +**Note on `is_publish`:** +- `is_publish` is simply a switch to control automatic publishing +- When `true` (default): Message is published to NATS automatically +- When `false`: Returns `(env, env_json_str)` without publishing, allowing manual publishing +- Connection reuse is achieved separately by creating NATS client outside the function + ### Julia Implementation #### Dependencies diff --git a/docs/implementation.md b/docs/implementation.md index 8da6ce3..304877b 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -338,6 +338,7 @@ node test/scenario3_julia_to_julia.js **Why the Difference?** - Julia supports function overloading and keyword arguments, allowing `NATS_connection` to be passed as an optional parameter - JavaScript/Python use a simpler `is_publish` option to control automatic publishing +- `is_publish` is simply a switch: when `true`, publish automatically; when `false`, return `(env, env_json_str)` without publishing - For connection reuse in JavaScript/Python, create a NATS client once and reuse it in your custom `fileserver_upload_handler` or custom publish logic ## Usage diff --git a/src/NATSBridge.js b/src/NATSBridge.js index 64b1c91..2bb776d 100644 --- a/src/NATSBridge.js +++ b/src/NATSBridge.js @@ -9,16 +9,16 @@ * File Server Handler Architecture: * The system uses handler functions to abstract file server operations, allowing support * for different file server implementations (e.g., Plik, AWS S3, custom HTTP server). - * + * * Handler Function Signatures: - * + * * ```javascript * // Upload handler - uploads data to file server and returns URL * // The handler is passed to smartsend as fileserverUploadHandler parameter * // It receives: (fileserver_url, dataname, data) * // Returns: { status, uploadid, fileid, url } - * async function fileserverUploadHandler(fileserver_url, dataname, data) { ... } - * + * async function plik_oneshot_upload(fileserver_url, dataname, data) { ... } + * * // Download handler - fetches data from file server URL with exponential backoff * // The handler is passed to smartreceive as fileserverDownloadHandler parameter * // It receives: (url, max_retries, base_delay, max_delay, correlation_id) @@ -213,73 +213,16 @@ function _deserialize_data(data, type, correlation_id) { } // Helper: Upload data to file server +// Internal wrapper that adds correlation_id logging for smartsend async function _upload_to_fileserver(fileserver_url, dataname, data, correlation_id) { /** - * Upload data to HTTP file server (plik-like API) - * - * This function implements the plik one-shot upload mode: - * 1. Creates a one-shot upload session by sending POST request with {"OneShot": true} - * 2. Uploads the file data as multipart form data - * 3. Returns identifiers and download URL for the uploaded file + * Internal upload helper - wraps plik_oneshot_upload to add correlation_id logging + * This allows smartsend to pass correlation_id for tracing without changing the handler signature */ log_trace(correlation_id, `Uploading ${dataname} to fileserver: ${fileserver_url}`); - - // Step 1: Get upload ID and token - const url_getUploadID = `${fileserver_url}/upload`; - const headers = { - "Content-Type": "application/json" - }; - const body = JSON.stringify({ OneShot: true }); - - let response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - if (!response.ok) { - throw new Error(`Failed to get upload ID: ${response.status} ${response.statusText}`); - } - - const responseJson = await response.json(); - const uploadid = responseJson.id; - const uploadtoken = responseJson.uploadToken; - - // Step 2: Upload file data - const url_upload = `${fileserver_url}/file/${uploadid}`; - - // Create multipart form data - const formData = new FormData(); - // Create a Blob from the Uint8Array - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - response = await fetch(url_upload, { - method: "POST", - headers: { - "X-UploadToken": uploadtoken - }, - body: formData - }); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.status} ${response.statusText}`); - } - - const fileResponseJson = await response.json(); - const fileid = fileResponseJson.id; - - // Build the download URL - const url = `${fileserver_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - log_trace(correlation_id, `Uploaded to URL: ${url}`); - - return { - status: response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; + const result = await plik_oneshot_upload(fileserver_url, dataname, data); + log_trace(correlation_id, `Uploaded to URL: ${result.url}`); + return result; } // Helper: Fetch data from URL with exponential backoff @@ -481,11 +424,14 @@ async function smartsend(subject, data, options = {}) { * @param {string} options.receiver_name - Name of the receiver (default: "") * @param {string} options.receiver_id - UUID of the receiver (default: "") * @param {string} options.reply_to - Topic to reply to (default: "") - * @param {string} options.reply_to_msg_id - Message ID this message is replying to (default: "") - * @param {boolean} options.is_publish - Whether to automatically publish the message to NATS (default: true) - * - * @returns {Promise} - A tuple-like object with { env: MessageEnvelope, env_json_str: string } - */ + * @param {string} options.reply_to_msg_id - Message ID this message is replying to (default: "") + * @param {boolean} options.is_publish - Whether to automatically publish the message to NATS (default: true) + * - When true: Message is published to NATS automatically + * - When false: Returns (env, env_json_str) without publishing, allowing manual publishing + * @returns {Promise} - A tuple-like object with { env: MessageEnvelope, env_json_str: string } + * - env: MessageEnvelope object with all metadata and payloads + * - env_json_str: JSON string representation of the envelope for manual publishing + */ const { broker_url = DEFAULT_NATS_URL, fileserver_url = DEFAULT_FILESERVER_URL, @@ -541,8 +487,8 @@ async function smartsend(subject, data, options = {}) { // Link path - Upload to HTTP server, send URL via NATS log_trace(correlation_id, `Using link transport, uploading to fileserver`); - // Upload to HTTP server - const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes, correlation_id); + // Upload to HTTP server using plik_oneshot_upload handler + const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes); if (response.status !== 200) { throw new Error(`Failed to upload data to fileserver: ${response.status}`); @@ -705,13 +651,18 @@ async function smartreceive(msg, options = {}) { } // plik_oneshot_upload - matches Julia plik_oneshot_upload function +// Upload handler signature: plik_oneshot_upload(fileserver_url, dataname, data) +// Returns: { status, uploadid, fileid, url } async function plik_oneshot_upload(file_server_url, dataname, data) { /** * Upload a single file to a plik server using one-shot mode * This function uploads raw byte array to a plik server in one-shot mode (no upload session). * It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, * retrieves an upload ID and token, then uploads the file data as multipart form data using the token. - * + * + * This is the default upload handler used by smartsend. + * Custom handlers can be passed via the fileserver_upload_handler option. + * * @param {string} file_server_url - Base URL of the plik server (e.g., "http://localhost:8080") * @param {string} dataname - Name of the file being uploaded * @param {Uint8Array} data - Raw byte data of the file content -- 2.49.1 From bee9f783d912d5839ab665302d6f20039a9269d9 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 17:38:50 +0700 Subject: [PATCH 32/35] update --- src/nats_bridge.py | 49 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/src/nats_bridge.py b/src/nats_bridge.py index 2b4c457..ef2b726 100644 --- a/src/nats_bridge.py +++ b/src/nats_bridge.py @@ -14,8 +14,24 @@ API Standard: # Input format for smartsend (always a list of tuples with type info) [(dataname1, data1, type1), (dataname2, data2, type2), ...] - # Output format for smartreceive (always returns a list of tuples) - [(dataname1, data1, type1), (dataname2, data2, type2), ...] + # Output format for smartreceive (returns a dictionary with payloads field containing list of tuples) + # Returns: Dict with envelope metadata and payloads field containing list of tuples + # { + # "correlation_id": "...", + # "msg_id": "...", + # "timestamp": "...", + # "send_to": "...", + # "msg_purpose": "...", + # "sender_name": "...", + # "sender_id": "...", + # "receiver_name": "...", + # "receiver_id": "...", + # "reply_to": "...", + # "reply_to_msg_id": "...", + # "broker_url": "...", + # "metadata": {...}, + # "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] + # } """ import json @@ -121,7 +137,7 @@ class MessageEnvelope: def __init__(self, send_to, payloads, correlation_id="", msg_id="", timestamp="", msg_purpose="", sender_name="", sender_id="", receiver_name="", - receiver_id="", reply_to="", reply_to_msg_id="", broker_url=DEFAULT_NATS_URL, + receiver_id="", reply_to="", reply_to_msg_id="", broker_url=DEFAULT_BROKER_URL, metadata=None): """ Initialize a MessageEnvelope. @@ -572,12 +588,18 @@ def smartsend(subject, data, broker_url=DEFAULT_BROKER_URL, fileserver_url=DEFAU publishes directly over NATS. Otherwise, it uploads the data to a fileserver and publishes only the download URL over NATS. + API Standard: + - Input format: List of (dataname, data, payload_type) tuples + - Even single payloads must be wrapped in a list + - Each payload can have a different type, enabling mixed-content messages + Args: subject: NATS subject to publish the message to data: List of (dataname, data, payload_type) tuples to send - dataname: Name of the payload - data: The actual data to send - payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") + - Example: [("message", "Hello World!", "text"), ("config", {"key": "value"}, "dictionary")] broker_url: URL of the NATS server fileserver_url: URL of the HTTP file server fileserver_upload_handler: Function to handle fileserver uploads (must return dict with "status", "uploadid", "fileid", "url" keys) @@ -711,6 +733,11 @@ def smartreceive(msg, fileserver_download_handler=_fetch_with_backoff, max_retri This function processes incoming NATS messages, handling both direct transport (base64 decoded payloads) and link transport (URL-based payloads). + API Standard: + - Returns a dictionary with envelope metadata and 'payloads' field + - payloads field contains list of (dataname, data, payload_type) tuples + - Supports mixed-content messages with different payload types + Args: msg: NATS message to process (dict or JSON string with envelope data) fileserver_download_handler: Function to handle downloading data from file server URLs @@ -723,6 +750,10 @@ def smartreceive(msg, fileserver_download_handler=_fetch_with_backoff, max_retri Returns: dict: Envelope dictionary with metadata and 'payloads' field containing list of (dataname, data, payload_type) tuples + - Envelope fields: correlation_id, msg_id, timestamp, send_to, msg_purpose, + sender_name, sender_id, receiver_name, receiver_id, reply_to, reply_to_msg_id, + broker_url, metadata + - payloads: List of (dataname, data, payload_type) tuples Example: >>> env = smartreceive(msg) @@ -821,10 +852,20 @@ if __name__ == "__main__": print(" from nats_bridge import smartsend, smartreceive") print() print(" # Send data (list of (dataname, data, payload_type) tuples)") + print(" # Even single payloads must be wrapped in a list") print(" data = [(\"message\", \"Hello World!\", \"text\")]") print(" env, env_json_str = smartsend(\"my.subject\", data)") print() print(" # On receiver:") print(" env = smartreceive(msg)") + print(" # env contains envelope metadata and payloads field") print(" for dataname, data, payload_type in env[\"payloads\"]:") - print(" print(\"Received {} of type {}: {}\".format(dataname, payload_type, data))") \ No newline at end of file + print(" print(\"Received {} of type {}: {}\".format(dataname, payload_type, data))") + print() + print(" # Mixed-content message example:") + print(" mixed_data = [") + print(" (\"text\", \"Hello!\", \"text\"),") + print(" (\"config\", {\"key\": \"value\"}, \"dictionary\"),") + print(" (\"table\", [{\"id\": 1}], \"table\")") + print(" ]") + print(" smartsend(\"/chat\", mixed_data)") \ No newline at end of file -- 2.49.1 From f8d93991f5190e40ed94516224fb9b0562075319 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 20:27:51 +0700 Subject: [PATCH 33/35] update --- README.md | 552 +--------- docs/architecture.md | 436 +------- docs/implementation.md | 580 +--------- examples/tutorial.md | 406 +------ examples/walkthrough.md | 1338 ++++++++--------------- test/test_js_dict_receiver.js | 80 -- test/test_js_dict_sender.js | 165 --- test/test_js_file_receiver.js | 71 -- test/test_js_file_sender.js | 144 --- test/test_js_mix_payload_sender.js | 277 ----- test/test_js_mix_payloads_receiver.js | 173 --- test/test_js_table_receiver.js | 87 -- test/test_js_table_sender.js | 165 --- test/test_js_text_receiver.js | 81 -- test/test_js_text_sender.js | 141 --- test/test_micropython_basic.py | 207 ---- test/test_micropython_dict_receiver.py | 70 -- test/test_micropython_dict_sender.py | 100 -- test/test_micropython_file_receiver.py | 65 -- test/test_micropython_file_sender.py | 80 -- test/test_micropython_mixed_receiver.py | 97 -- test/test_micropython_mixed_sender.py | 94 -- test/test_micropython_text_receiver.py | 69 -- test/test_micropython_text_sender.py | 82 -- 24 files changed, 579 insertions(+), 4981 deletions(-) delete mode 100644 test/test_js_dict_receiver.js delete mode 100644 test/test_js_dict_sender.js delete mode 100644 test/test_js_file_receiver.js delete mode 100644 test/test_js_file_sender.js delete mode 100644 test/test_js_mix_payload_sender.js delete mode 100644 test/test_js_mix_payloads_receiver.js delete mode 100644 test/test_js_table_receiver.js delete mode 100644 test/test_js_table_sender.js delete mode 100644 test/test_js_text_receiver.js delete mode 100644 test/test_js_text_sender.js delete mode 100644 test/test_micropython_basic.py delete mode 100644 test/test_micropython_dict_receiver.py delete mode 100644 test/test_micropython_dict_sender.py delete mode 100644 test/test_micropython_file_receiver.py delete mode 100644 test/test_micropython_file_sender.py delete mode 100644 test/test_micropython_mixed_receiver.py delete mode 100644 test/test_micropython_mixed_sender.py delete mode 100644 test/test_micropython_text_receiver.py delete mode 100644 test/test_micropython_text_sender.py diff --git a/README.md b/README.md index 7e0fc89..e5885a7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NATSBridge -A high-performance, bi-directional data bridge between **Julia**, **JavaScript**, and **Python/Micropython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +A high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![NATS](https://img.shields.io/badge/NATS-Enabled-green.svg)](https://nats.io) @@ -25,7 +25,7 @@ A high-performance, bi-directional data bridge between **Julia**, **JavaScript** ## Overview -NATSBridge enables seamless communication across Julia, JavaScript, and Python/Micropython applications through NATS, with intelligent transport selection based on payload size: +NATSBridge enables seamless communication for Julia applications through NATS, with intelligent transport selection based on payload size: | Transport | Payload Size | Method | |-----------|--------------|--------| @@ -37,14 +37,13 @@ NATSBridge enables seamless communication across Julia, JavaScript, and Python/M - **Chat Applications**: Text, images, audio, video in a single message - **File Transfer**: Efficient transfer of large files using claim-check pattern - **Streaming Data**: Sensor data, telemetry, and analytics pipelines -- **Cross-Platform Communication**: Julia ↔ JavaScript ↔ Python/Micropython -- **IoT Devices**: Micropython devices sending data to cloud services + --- ## Features -- ✅ **Bi-directional messaging** between Julia, JavaScript, and Python/Micropython +- ✅ **Bi-directional messaging** for Julia applications - ✅ **Multi-payload support** - send multiple payloads with different types in one message - ✅ **Automatic transport selection** - direct vs link based on payload size - ✅ **Claim-Check pattern** for payloads > 1MB @@ -53,7 +52,7 @@ NATSBridge enables seamless communication across Julia, JavaScript, and Python/M - ✅ **Correlation ID tracking** for message tracing - ✅ **Reply-to support** for request-response patterns - ✅ **JetStream support** for message replay and durability -- ✅ **Lightweight Micropython implementation** for microcontrollers + --- @@ -65,16 +64,12 @@ NATSBridge enables seamless communication across Julia, JavaScript, and Python/M ┌─────────────────────────────────────────────────────────────────────┐ │ NATSBridge Architecture │ ├─────────────────────────────────────────────────────────────────────┤ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Julia │ │ JavaScript │ │ Python/ │ │ -│ │ (NATS.jl) │◄──►│ (nats.js) │◄──►│ Micropython │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌─────────────────────────────────────────────────────┐ │ -│ │ NATS │ │ -│ │ (Message Broker) │ │ -│ └─────────────────────────────────────────────────────┘ │ +│ ┌──────────────┐ │ │ +│ │ Julia │ ▼ │ +│ │ (NATS.jl) │ ┌─────────────────────────┐ │ +│ └──────────────┘ │ NATS │ │ +│ │ (Message Broker) │ │ +│ └─────────────────────────┘ │ │ │ │ │ ▼ │ │ ┌──────────────────────┐ │ @@ -110,40 +105,6 @@ Pkg.add("NATS") Pkg.add("https://git.yiem.cc/ton/NATSBridge") ``` -### JavaScript - -```bash -npm install nats.js apache-arrow uuid base64-url -``` - -For Node.js: -```javascript -const { smartsend, smartreceive } = require('./src/NATSBridge'); -``` - -For browser: -```html - - -``` - -### Python/Micropython - -1. Copy [`src/nats_bridge.py`](src/nats_bridge.py) to your device -2. Install dependencies: - -**For Python (desktop):** -```bash -pip install nats-py -``` - -**For Micropython:** -- `urequests` for HTTP requests (built-in for ESP32) -- `base64` for base64 encoding (built-in) -- `json` for JSON handling (built-in) - --- ## Quick Start @@ -160,36 +121,12 @@ docker run -p 4222:4222 nats:latest # Create a directory for file uploads mkdir -p /tmp/fileserver -# Use Python's built-in server +# Start HTTP file server python3 -m http.server 8080 --directory /tmp/fileserver ``` ### Step 3: Send Your First Message -#### Python/Micropython - -```python -from nats_bridge import smartsend - -# Send a text message -data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") -print("Message sent!") -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Send a text message -const { env, env_json_str } = await smartsend("/chat/room1", [ - { dataname: "message", data: "Hello World", type: "text" } -], { broker_url: "nats://localhost:4222" }); - -console.log("Message sent!"); -``` - #### Julia ```julia @@ -203,64 +140,6 @@ println("Message sent!") ### Step 4: Receive Messages -#### Python/Micropython - -```python -import nats -import asyncio -from nats_bridge import smartreceive - -# Configuration -SUBJECT = "/chat/room1" -NATS_URL = "nats://localhost:4222" - -async def main(): - # Connect to NATS - nc = await nats.connect(NATS_URL) - - # Subscribe to the subject - msg comes from the callback - async def message_handler(msg): - # Receive and process message - env = smartreceive(msg.data) - for dataname, data, type in env["payloads"]: - print(f"Received {dataname}: {data}") - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - await asyncio.sleep(120) # Keep listening - await nc.close() - -asyncio.run(main()) -``` - -#### JavaScript - -```javascript -const { smartreceive } = require('./src/NATSBridge'); -const { connect } = require('nats'); - -// Configuration -const SUBJECT = "/chat/room1"; -const NATS_URL = "nats://localhost:4222"; - -async function main() { - // Connect to NATS - const nc = await connect({ servers: [NATS_URL] }); - - // Subscribe to the subject - msg comes from the async iteration - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - // Receive and process message - const env = await smartreceive(msg); - for (const payload of env.payloads) { - console.log(`Received ${payload.dataname}: ${payload.data}`); - } - } -} - -main(); -``` - #### Julia ```julia @@ -305,54 +184,6 @@ test_receive() Sends data either directly via NATS or via a fileserver URL, depending on payload size. -#### Python/Micropython - -```python -from nats_bridge import smartsend - -env, env_json_str = smartsend( - subject, # NATS subject to publish to - data, # List of (dataname, data, type) tuples - broker_url="nats://localhost:4222", # NATS server URL - fileserver_url="http://localhost:8080", # File server URL - fileserver_upload_handler=plik_oneshot_upload, # Upload handler function - size_threshold=1_000_000, # Threshold in bytes (default: 1MB) - correlation_id=None, # Optional correlation ID for tracing - msg_purpose="chat", # Message purpose - sender_name="NATSBridge", # Sender name - receiver_name="", # Receiver name (empty = broadcast) - receiver_id="", # Receiver UUID (empty = broadcast) - reply_to="", # Reply topic - reply_to_msg_id="", # Reply message ID - is_publish=True # Whether to automatically publish to NATS -) -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const { env, env_json_str } = await smartsend( - subject, // NATS subject - data, // Array of {dataname, data, type} - { - broker_url: "nats://localhost:4222", - fileserver_url: "http://localhost:8080", - fileserver_upload_handler: customUploadHandler, - size_threshold: 1_000_000, - correlation_id: "custom-id", - msg_purpose: "chat", - sender_name: "NATSBridge", - receiver_name: "", - receiver_id: "", - reply_to: "", - reply_to_msg_id: "", - is_publish: true // Whether to automatically publish to NATS - } -); -``` - #### Julia ```julia @@ -384,40 +215,6 @@ env, env_json_str = NATSBridge.smartsend( Receives and processes messages from NATS, handling both direct and link transport. -#### Python/Micropython - -```python -from nats_bridge import smartreceive - -# Note: For nats-py, use msg.data to pass the raw message data -env = smartreceive( - msg.data, # NATS message data (msg.data for nats-py) - fileserver_download_handler=_fetch_with_backoff, # Download handler - max_retries=5, # Max retry attempts - base_delay=100, # Initial delay in ms - max_delay=5000 # Max delay in ms -) -# Returns: Dict with envelope metadata and 'payloads' field -``` - -#### JavaScript - -```javascript -const { smartreceive } = require('./src/NATSBridge'); - -// Note: msg is the NATS message object from subscription -const env = await smartreceive( - msg, // NATS message (raw object from subscription) - { - fileserverDownloadHandler: customDownloadHandler, - maxRetries: 5, - baseDelay: 100, - maxDelay: 5000 - } -); -// Returns: Object with envelope metadata and payloads array -``` - #### Julia ```julia @@ -485,19 +282,6 @@ NATSBridge.publish_message(conn, "/chat/room1", "{\"correlation_id\":\"abc123\"} Small payloads are sent directly via NATS with Base64 encoding. -#### Python/Micropython -```python -data = [("message", "Hello", "text")] -smartsend("/topic", data) -``` - -#### JavaScript -```javascript -await smartsend("/topic", [ - { dataname: "message", data: "Hello", type: "text" } -]); -``` - #### Julia ```julia data = [("message", "Hello", "text")] @@ -508,19 +292,6 @@ smartsend("/topic", data) Large payloads are uploaded to an HTTP file server. -#### Python/Micropython -```python -data = [("file", large_data, "binary")] -smartsend("/topic", data, fileserver_url="http://localhost:8080") -``` - -#### JavaScript -```javascript -await smartsend("/topic", [ - { dataname: "file", data: largeData, type: "binary" } -], { fileserver_url: "http://localhost:8080" }); -``` - #### Julia ```julia data = [("file", large_data, "binary")] @@ -531,38 +302,10 @@ smartsend("/topic", data; fileserver_url="http://localhost:8080") ## Examples -All examples include code for **Julia**, **JavaScript**, and **Python/Micropython** unless otherwise specified. - ### Example 1: Chat with Mixed Content Send text, small image, and large file in one message. -#### Python/Micropython -```python -from nats_bridge import smartsend - -data = [ - ("message_text", "Hello!", "text"), - ("user_avatar", image_data, "image"), - ("large_document", large_file_data, "binary") -] - -env, env_json_str = smartsend("/chat/room1", data, fileserver_url="http://localhost:8080") -``` - -#### JavaScript -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const { env, env_json_str } = await smartsend("/chat/room1", [ - { dataname: "message_text", data: "Hello!", type: "text" }, - { dataname: "user_avatar", data: image_data, type: "image" }, - { dataname: "large_document", data: large_file_data, type: "binary" } -], { - fileserver_url: "http://localhost:8080" -}); -``` - #### Julia ```julia using NATSBridge @@ -580,35 +323,6 @@ env, env_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="ht Send configuration data between platforms. -#### Python/Micropython -```python -from nats_bridge import smartsend - -config = { - "wifi_ssid": "MyNetwork", - "wifi_password": "password123", - "update_interval": 60 -} - -data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data) -``` - -#### JavaScript -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const config = { - wifi_ssid: "MyNetwork", - wifi_password: "password123", - update_interval: 60 -}; - -const { env, env_json_str } = await smartsend("/device/config", [ - { dataname: "config", data: config, type: "dictionary" } -]); -``` - #### Julia ```julia using NATSBridge @@ -627,36 +341,6 @@ env, env_json_str = NATSBridge.smartsend("/device/config", data) Send tabular data using Apache Arrow IPC format. -#### Python/Micropython -```python -import pandas as pd -from nats_bridge import smartsend - -df = pd.DataFrame({ - "id": [1, 2, 3], - "name": ["Alice", "Bob", "Charlie"], - "score": [95, 88, 92] -}) - -data = [("students", df, "table")] -env, env_json_str = smartsend("/data/analysis", data) -``` - -#### JavaScript -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const tableData = [ - { id: 1, name: "Alice", score: 95 }, - { id: 2, name: "Bob", score: 88 }, - { id: 3, name: "Charlie", score: 92 } -]; - -const { env, env_json_str } = await smartsend("/data/analysis", [ - { dataname: "students", data: tableData, type: "table" } -]); -``` - #### Julia ```julia using NATSBridge @@ -676,106 +360,6 @@ env, env_json_str = NATSBridge.smartsend("/data/analysis", data) Bi-directional communication with reply-to support. The `smartsend` function now returns both the envelope object and a JSON string that can be published directly. -#### Python/Micropython (Requester) -```python -from nats_bridge import smartsend - -env, env_json_str = smartsend( - "/device/command", - [("command", {"action": "read_sensor"}, "dictionary")], - broker_url="nats://localhost:4222", - reply_to="/device/response" -) -# env: MessageEnvelope object -# env_json_str: JSON string for publishing to NATS - -# The env_json_str can also be published directly using NATS request-reply pattern -# nc.request("/device/command", env_json_str, reply_to="/device/response") -``` - -#### Python/Micropython (Responder) -```python -import nats -import asyncio -from nats_bridge import smartreceive, smartsend - -# Configuration -SUBJECT = "/device/command" -NATS_URL = "nats://localhost:4222" - -async def main(): - nc = await nats.connect(NATS_URL) - - async def message_handler(msg): - # Receive and parse the incoming message envelope - env = smartreceive(msg.data) - - # Extract reply_to from the envelope metadata - reply_to = env["reply_to"] - - for dataname, data, type in env["payloads"]: - if data.get("action") == "read_sensor": - response = {"sensor_id": "sensor-001", "value": 42.5} - # Send response to the reply_to subject from the request - if reply_to: - smartsend(reply_to, [("data", response, "dictionary")]) - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - await asyncio.sleep(120) - await nc.close() - -asyncio.run(main()) -``` - -#### JavaScript (Requester) -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const { env, env_json_str } = await smartsend("/device/command", [ - { dataname: "command", data: { action: "read_sensor" }, type: "dictionary" } -], { - broker_url: "nats://localhost:4222", - reply_to: "/device/response" -}); -``` - -#### JavaScript (Responder) -```javascript -const { smartreceive, smartsend } = require('./src/NATSBridge'); -const { connect } = require('nats'); - -// Configuration -const SUBJECT = "/device/command"; -const NATS_URL = "nats://localhost:4222"; - -async function main() { - const nc = await connect({ servers: [NATS_URL] }); - - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - const env = await smartreceive(msg); - - // Extract reply_to from the envelope metadata - const replyTo = env["reply_to"]; - - for (const payload of env.payloads) { - if (payload.dataname === "command" && payload.data.action === "read_sensor") { - const response = { sensor_id: "sensor-001", value: 42.5 }; - // Send response to the reply_to subject from the request - if (replyTo) { - await smartsend(replyTo, [ - { dataname: "data", data: response, type: "dictionary" } - ]); - } - } - } - } -} - -main(); -``` - #### Julia (Requester) ```julia using NATSBridge @@ -822,70 +406,9 @@ end test_responder() ``` -### Example 5: Micropython IoT Device +### Example 5: IoT Device Sensor Data -Lightweight Micropython device sending sensor data. - -#### Micropython -```python -import nats -import asyncio -from nats_bridge import smartsend, smartreceive - -# Configuration -SUBJECT = "/device/sensors" -NATS_URL = "nats://localhost:4222" - -async def main(): - nc = await nats.connect(NATS_URL) - - # Send sensor data - data = [("temperature", "25.5", "text"), ("humidity", 65, "dictionary")] - smartsend("/device/sensors", data, broker_url="nats://localhost:4222") - - # Receive commands - msg comes from the callback - async def message_handler(msg): - env = smartreceive(msg.data) - for dataname, data, type in env["payloads"]: - if type == "dictionary" and data.get("action") == "reboot": - # Execute reboot - pass - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - await asyncio.sleep(120) - await nc.close() - -asyncio.run(main()) -``` - -#### JavaScript (Receiver) -```javascript -const { smartreceive } = require('./src/NATSBridge'); -const { connect } = require('nats'); - -// Configuration -const SUBJECT = "/device/sensors"; -const NATS_URL = "nats://localhost:4222"; - -async function main() { - const nc = await connect({ servers: [NATS_URL] }); - - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - const env = await smartreceive(msg); - for (const payload of env.payloads) { - if (payload.dataname === "temperature") { - console.log(`Temperature: ${payload.data}`); - } else if (payload.dataname === "humidity") { - console.log(`Humidity: ${payload.data}`); - } - } - } -} - -main(); -``` +IoT device sending sensor data. #### Julia (Receiver) ```julia @@ -921,53 +444,6 @@ test_receiver() Run the test scripts to verify functionality: -### Python/Micropython - -```bash -# Basic functionality test -python test/test_micropython_basic.py - -# Text message exchange -python test/test_micropython_text_sender.py -python test/test_micropython_text_receiver.py - -# Dictionary exchange -python test/test_micropython_dict_sender.py -python test/test_micropython_dict_receiver.py - -# File transfer -python test/test_micropython_file_sender.py -python test/test_micropython_file_receiver.py - -# Mixed payload types -python test/test_micropython_mixed_sender.py -python test/test_micropython_mixed_receiver.py -``` - -### JavaScript - -```bash -# Text message exchange -node test/test_js_text_sender.js -node test/test_js_text_receiver.js - -# Dictionary exchange -node test/test_js_dict_sender.js -node test/test_js_dict_receiver.js - -# File transfer -node test/test_js_file_sender.js -node test/test_js_file_receiver.js - -# Mixed payload types -node test/test_js_mix_payload_sender.js -node test/test_js_mix_payloads_receiver.js - -# Table exchange -node test/test_js_table_sender.js -node test/test_js_table_receiver.js -``` - ### Julia ```julia diff --git a/docs/architecture.md b/docs/architecture.md index 4a5f29b..47891e3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,12 +2,10 @@ ## Overview -This document describes the architecture for a high-performance, bi-directional data bridge between **Julia**, **JavaScript**, and **Python/Micropython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +This document describes the architecture for a high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. -The system enables seamless communication across all three platforms: -- **Julia ↔ JavaScript** bi-directional messaging -- **JavaScript ↔ Python/Micropython** bi-directional messaging -- **Julia ↔ Python/Micropython** bi-directional messaging (via JSON serialization) +The system enables seamless communication for Julia applications: +- **Julia** messaging with NATS ### File Server Handler Architecture @@ -113,8 +111,7 @@ env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_ret ```mermaid flowchart TD subgraph Client - JS[JavaScript Client] - JSApp[Application Logic] + App[Julia Application] end subgraph Server @@ -123,14 +120,12 @@ flowchart TD FileServer[HTTP File Server] end - JS -->|Control/Small Data| JSApp - JSApp -->|NATS| NATS + App -->|NATS| NATS NATS -->|NATS| Julia Julia -->|NATS| NATS Julia -->|HTTP POST| FileServer - JS -->|HTTP GET| FileServer - style JS fill:#e1f5fe + style App fill:#e8f5e9 style Julia fill:#e8f5e9 style NATS fill:#fff3e0 style FileServer fill:#f3e5f5 @@ -140,7 +135,7 @@ flowchart TD ### 1. msg_envelope_v1 - Message Envelope -The `msg_envelope_v1` structure provides a comprehensive message format for bidirectional communication between Julia, JavaScript, and Python/Micropython applications. +The `msg_envelope_v1` structure provides a comprehensive message format for bidirectional communication in Julia applications. **Julia Structure:** ```julia @@ -216,7 +211,7 @@ end ### 2. msg_payload_v1 - Payload Structure -The `msg_payload_v1` structure provides flexible payload handling for various data types across all supported platforms. +The `msg_payload_v1` structure provides flexible payload handling for various data types. **Julia Structure:** ```julia @@ -271,65 +266,7 @@ end └─────────────────┘ └─────────────────┘ ``` -### 4. Cross-Platform Architecture - -```mermaid -flowchart TD - subgraph PythonMicropython - Py[Python/Micropython] - PySmartSend[smartsend] - PySmartReceive[smartreceive] - end - - subgraph JavaScript - JS[JavaScript] - JSSmartSend[smartsend] - JSSmartReceive[smartreceive] - end - - subgraph Julia - Julia[Julia] - JuliaSmartSend[smartsend] - JuliaSmartReceive[smartreceive] - end - - subgraph NATS - NATSServer[NATS Server] - end - - PySmartSend --> NATSServer - JSSmartSend --> NATSServer - JuliaSmartSend --> NATSServer - NATSServer --> PySmartReceive - NATSServer --> JSSmartReceive - NATSServer --> JuliaSmartReceive - - style PythonMicropython fill:#e1f5fe - style JavaScript fill:#f3e5f5 - style Julia fill:#e8f5e9 -``` - -### 5. Python/Micropython Module Architecture - -```mermaid -graph TD - subgraph PyModule - PySmartSend[smartsend] - SizeCheck[Size Check] - DirectPath[Direct Path] - LinkPath[Link Path] - HTTPClient[HTTP Client] - end - - PySmartSend --> SizeCheck - SizeCheck -->|< 1MB| DirectPath - SizeCheck -->|>= 1MB| LinkPath - LinkPath --> HTTPClient - - style PyModule fill:#b3e5fc -``` - -### 6. Julia Module Architecture +### 4. Julia Module Architecture ```mermaid graph TD @@ -349,51 +286,8 @@ graph TD style JuliaModule fill:#c5e1a5 ``` -### 7. JavaScript Module Architecture - -```mermaid -graph TD - subgraph JSModule - JSSmartSend[smartsend] - JSSmartReceive[smartreceive] - JetStreamConsumer[JetStream Pull Consumer] - ApacheArrow[Apache Arrow] - end - - JSSmartSend --> NATS - JSSmartReceive --> JetStreamConsumer - JetStreamConsumer --> ApacheArrow - - style JSModule fill:#f3e5f5 -``` - ## Implementation Details -### API Consistency Across Languages - -**High-Level API (Consistent Across All Languages):** -- `smartsend(subject, data, ...)` - Main publishing function -- `smartreceive(msg, ...)` - Main receiving function -- Message envelope structure (`msg_envelope_v1` / `MessageEnvelope`) -- Payload structure (`msg_payload_v1` / `MessagePayload`) -- Transport strategy (direct vs link based on size threshold) -- Supported payload types: text, dictionary, table, image, audio, video, binary - -**Low-Level Native Functions (Language-Specific Conventions):** -- Julia: `NATS.connect()`, `publish_message()`, function overloading -- JavaScript: `nats.js` client, native async/await patterns -- Python: `nats-python` client, native async/await patterns - -**Connection Reuse Pattern:** -- **Julia:** Uses `NATS_connection` keyword parameter with function overloading -- **JavaScript/Python:** Achieved by creating NATS client outside the function and reusing it in custom handlers - -**Note on `is_publish`:** -- `is_publish` is simply a switch to control automatic publishing -- When `true` (default): Message is published to NATS automatically -- When `false`: Returns `(env, env_json_str)` without publishing, allowing manual publishing -- Connection reuse is achieved separately by creating NATS client outside the function - ### Julia Implementation #### Dependencies @@ -546,200 +440,6 @@ env, env_json_str = smartsend( # Uses: publish_message(broker_url, subject, env_json_str, cid) ``` -**API Consistency Note:** -- **High-level API (smartsend, smartreceive):** Uses consistent naming across all three languages (Julia, JavaScript, Python/Micropython) -- **Low-level native functions (NATS.connect(), publish_message()):** Follow the conventions of the specific language ecosystem and do not require cross-language consistency - -### JavaScript Implementation - -#### Dependencies -- `nats.js` - Core NATS functionality -- `apache-arrow` - Arrow IPC serialization -- `uuid` - Correlation ID and message ID generation -- `base64-arraybuffer` - Base64 encoding/decoding -- `node-fetch` or `fetch` - HTTP client for file server - -#### smartsend Function - -```javascript -async function smartsend( - subject, - data, // List of (dataname, data, type) tuples: [(dataname1, data1, type1), ...] - options = {} -) -``` - -**Options:** -- `broker_url` (String) - NATS server URL (default: `"nats://localhost:4222"`) -- `fileserver_url` (String) - Base URL of the file server (default: `"http://localhost:8080"`) -- `size_threshold` (Number) - Threshold in bytes for transport selection (default: `1048576` = 1MB) -- `correlation_id` (String) - Optional correlation ID for tracing -- `msg_purpose` (String) - Purpose of the message (default: `"chat"`) -- `sender_name` (String) - Sender name (default: `"NATSBridge"`) -- `receiver_name` (String) - Message receiver name (default: `""`) -- `receiver_id` (String) - Message receiver ID (default: `""`) -- `reply_to` (String) - Topic to reply to (default: `""`) -- `reply_to_msg_id` (String) - Message ID this message is replying to (default: `""`) -- `is_publish` (Boolean) - Whether to automatically publish the message to NATS (default: `true`) -- `fileserver_upload_handler` (Function) - Custom upload handler function - -**Note:** JavaScript uses `is_publish` option (instead of `NATS_connection` keyword) to control automatic publishing behavior. Connection reuse can be achieved by creating a NATS client outside the function and reusing it in a custom `fileserver_upload_handler` or custom publish implementation. - -**Return Value:** -- Returns a Promise that resolves to an object containing: - - `env` - The envelope object containing all metadata and payloads - - `env_json_str` - JSON string representation of the envelope for publishing - -**Input Format:** -- `data` - **Must be a list of (dataname, data, type) tuples**: `[(dataname1, data1, "type1"), (dataname2, data2, "type2"), ...]` -- Even for single payloads: `[(dataname1, data1, "type1")]` -- Each payload can have a different type, enabling mixed-content messages -- Supported types: `"text"`, `"dictionary"`, `"table"`, `"image"`, `"audio"`, `"video"`, `"binary"` - -**Flow:** -1. Generate correlation ID and message ID if not provided -2. Iterate through the list of `(dataname, data, type)` tuples -3. For each payload: - - Serialize based on payload type - - Check payload size - - If < threshold: Base64 encode and include in envelope - - If >= threshold: Upload to HTTP server, store URL in envelope -4. Publish the JSON envelope to NATS -5. Return envelope object and JSON string - -#### smartreceive Handler - -```javascript -async function smartreceive(msg, options = {}) -``` - -**Options:** -- `fileserver_download_handler` (Function) - Custom download handler function -- `max_retries` (Number) - Maximum retry attempts for fetching URL (default: `5`) -- `base_delay` (Number) - Initial delay for exponential backoff in ms (default: `100`) -- `max_delay` (Number) - Maximum delay for exponential backoff in ms (default: `5000`) -- `correlation_id` (String) - Optional correlation ID for tracing - -**Output Format:** -- Returns a Promise that resolves to an object containing all envelope fields: - - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - - `metadata` - Message-level metadata dictionary - - `payloads` - List of tuples, each containing `(dataname, data, type)` with deserialized payload data - -**Process Flow:** -1. Parse the JSON envelope to extract all fields -2. Iterate through each payload in `payloads` array -3. For each payload: - - Determine transport type (`direct` or `link`) - - If `direct`: Base64 decode the data from the message - - If `link`: Fetch data from URL using exponential backoff (via `fileserver_download_handler`) - - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) -4. Return envelope object with `payloads` field containing list of `(dataname, data, type)` tuples - -**Note:** The `fileserver_download_handler` receives `(url, max_retries, base_delay, max_delay, correlation_id)` and returns `ArrayBuffer` or `Uint8Array`. - -### Python/Micropython Implementation - -#### Dependencies -- `nats-python` - Core NATS functionality -- `pyarrow` - Arrow IPC serialization -- `uuid` - Correlation ID and message ID generation -- `base64` - Base64 encoding/decoding -- `requests` or `aiohttp` - HTTP client for file server - -#### smartsend Function - -```python -def smartsend( - subject: str, - data: List[Tuple[str, Any, str]], # List of (dataname, data, type) tuples - broker_url: str = DEFAULT_BROKER_URL, - fileserver_url: str = DEFAULT_FILESERVER_URL, - fileserver_upload_handler: Callable = plik_oneshot_upload, - size_threshold: int = DEFAULT_SIZE_THRESHOLD, - correlation_id: Union[str, None] = None, - msg_purpose: str = "chat", - sender_name: str = "NATSBridge", - receiver_name: str = "", - receiver_id: str = "", - reply_to: str = "", - reply_to_msg_id: str = "", - is_publish: bool = True -) -> Tuple[MessageEnvelope, str] -``` - -**Options:** -- `broker_url` (str) - NATS server URL (default: `"nats://localhost:4222"`) -- `fileserver_url` (str) - Base URL of the file server (default: `"http://localhost:8080"`) -- `size_threshold` (int) - Threshold in bytes for transport selection (default: `1048576` = 1MB) -- `correlation_id` (str) - Optional correlation ID for tracing (auto-generated if None) -- `msg_purpose` (str) - Purpose of the message (default: `"chat"`) -- `sender_name` (str) - Sender name (default: `"NATSBridge"`) -- `receiver_name` (str) - Message receiver name (default: `""`) -- `receiver_id` (str) - Message receiver ID (default: `""`) -- `reply_to` (str) - Topic to reply to (default: `""`) -- `reply_to_msg_id` (str) - Message ID this message is replying to (default: `""`) -- `is_publish` (bool) - Whether to automatically publish the message to NATS (default: `True`) -- `fileserver_upload_handler` (Callable) - Custom upload handler function - -**Note:** Python uses `is_publish` parameter (instead of `NATS_connection` keyword) to control automatic publishing behavior. Connection reuse can be achieved by creating a NATS client outside the function and reusing it in a custom `fileserver_upload_handler` or custom publish implementation. - -**Return Value:** -- Returns a tuple `(env, env_json_str)` where: - - `env` - The envelope dictionary containing all metadata and payloads - - `env_json_str` - JSON string representation of the envelope for publishing - -**Input Format:** -- `data` - **Must be a list of (dataname, data, type) tuples**: `[(dataname1, data1, "type1"), (dataname2, data2, "type2"), ...]` -- Even for single payloads: `[(dataname1, data1, "type1")]` -- Each payload can have a different type, enabling mixed-content messages -- Supported types: `"text"`, `"dictionary"`, `"table"`, `"image"`, `"audio"`, `"video"`, `"binary"` - -**Flow:** -1. Generate correlation ID and message ID if not provided -2. Iterate through the list of `(dataname, data, type)` tuples -3. For each payload: - - Serialize based on payload type - - Check payload size - - If < threshold: Base64 encode and include in envelope - - If >= threshold: Upload to HTTP server, store URL in envelope -4. Publish the JSON envelope to NATS -5. Return envelope dictionary and JSON string - -#### smartreceive Handler - -```python -async def smartreceive( - msg: NATS.Message, - options: Dict = {} -) -``` - -**Options:** -- `fileserver_download_handler` (Callable) - Custom download handler function -- `max_retries` (int) - Maximum retry attempts for fetching URL (default: `5`) -- `base_delay` (int) - Initial delay for exponential backoff in ms (default: `100`) -- `max_delay` (int) - Maximum delay for exponential backoff in ms (default: `5000`) -- `correlation_id` (str) - Optional correlation ID for tracing - -**Output Format:** -- Returns a JSON object (dictionary) containing all envelope fields: - - `correlation_id`, `msg_id`, `timestamp`, `send_to`, `msg_purpose`, `sender_name`, `sender_id`, `receiver_name`, `receiver_id`, `reply_to`, `reply_to_msg_id`, `broker_url` - - `metadata` - Message-level metadata dictionary - - `payloads` - List of tuples, each containing `(dataname, data, payload_type)` with deserialized payload data - -**Process Flow:** -1. Parse the JSON envelope to extract all fields -2. Iterate through each payload in `payloads` list -3. For each payload: - - Determine transport type (`direct` or `link`) - - If `direct`: Base64 decode the data from the message - - If `link`: Fetch data from URL using exponential backoff (via `fileserver_download_handler`) - - Deserialize based on payload type (`dictionary`, `table`, `binary`, etc.) -4. Return envelope dictionary with `payloads` field containing list of `(dataname, data, type)` tuples - -**Note:** The `fileserver_download_handler` receives `(url: str, max_retries: int, base_delay: int, max_delay: int, correlation_id: str)` and returns `bytes`. - ## Scenario Implementations ### Scenario 1: Command & Control (Small Dictionary) @@ -752,18 +452,6 @@ async def smartreceive( # Send acknowledgment ``` -**JavaScript (Sender/Receiver):** -```javascript -// Create small dictionary config -// Send via smartsend with type="dictionary" -``` - -**Python/Micropython (Sender/Receiver):** -```python -# Create small dictionary config -# Send via smartsend with type="dictionary" -``` - ### Scenario 2: Deep Dive Analysis (Large Arrow Table) **Julia (Sender/Receiver):** @@ -775,32 +463,8 @@ async def smartreceive( # Publish NATS with URL ``` -**JavaScript (Sender/Receiver):** -```javascript -// Receive NATS message with URL -// Fetch data from HTTP server -// Parse Arrow IPC with zero-copy -// Load into Perspective.js or D3 -``` - -**Python/Micropython (Sender/Receiver):** -```python -# Create large DataFrame -# Convert to Arrow IPC stream -# Check size (> 1MB) -# Upload to HTTP server -# Publish NATS with URL -``` - ### Scenario 3: Live Audio Processing -**JavaScript (Sender/Receiver):** -```javascript -// Capture audio chunk -// Send as binary with metadata headers -// Use smartsend with type="audio" -``` - **Julia (Sender/Receiver):** ```julia # Receive audio data @@ -808,13 +472,6 @@ async def smartreceive( # Send results back (JSON + Arrow table) ``` -**Python/Micropython (Sender/Receiver):** -```python -# Capture audio chunk -# Send as binary with metadata headers -# Use smartsend with type="audio" -``` - ### Scenario 4: Catch-Up (JetStream) **Julia (Producer/Consumer):** @@ -823,22 +480,9 @@ async def smartreceive( # Include metadata for temporal tracking ``` -**JavaScript (Producer/Consumer):** -```javascript -// Connect to JetStream -// Request replay from last 10 minutes -// Process historical and real-time messages -``` - -**Python/Micropython (Producer/Consumer):** -```python -# Publish to JetStream -# Include metadata for temporal tracking -``` - ### Scenario 5: Selection (Low Bandwidth) -**Focus:** Small Arrow tables, cross-platform communication. The Action: Any platform wants to send a small DataFrame to show on any receiving application for the user to choose. +**Focus:** Small Arrow tables. The Action: Julia wants to send a small DataFrame to show on a receiving application for the user to choose. **Julia (Sender/Receiver):** ```julia @@ -849,30 +493,9 @@ async def smartreceive( # Include metadata for dashboard selection context ``` -**JavaScript (Sender/Receiver):** -```javascript -// Receive NATS message with direct transport -// Decode Base64 payload -// Parse Arrow IPC with zero-copy -// Load into selection UI component (e.g., dropdown, table) -// User makes selection -// Send selection back to Julia -``` - -**Python/Micropython (Sender/Receiver):** -```python -# Create small DataFrame (e.g., 50KB - 500KB) -# Convert to Arrow IPC stream -# Check payload size (< 1MB threshold) -# Publish directly to NATS with Base64-encoded payload -# Include metadata for dashboard selection context -``` - -**Use Case:** Any server generates a list of available options (e.g., file selections, configuration presets) as a small DataFrame and sends to any receiving application for user selection. The selection is then sent back to the sender for processing. - ### Scenario 6: Chat System -**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging across all platforms. +**Focus:** Every conversational message is composed of any number and any combination of components, spanning the full spectrum from small to large. This includes text, images, audio, video, tables, and files—specifically accommodating everything from brief snippets to high-resolution images, large audio files, extensive tables, and massive documents. Support for claim-check delivery and full bi-directional messaging. **Multi-Payload Support:** The system supports mixed-payload messages where a single message can contain multiple payloads with different transport strategies. The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. @@ -894,42 +517,7 @@ async def smartreceive( # Support bidirectional messaging with replyTo fields ``` -**JavaScript (Sender/Receiver):** -```javascript -// Build chat message with mixed content: -// - User input text: direct transport -// - Selected image: check size, use appropriate transport -// - Audio recording: link transport for large files -// - File attachment: link transport -// -// Parse received message: -// - Direct payloads: decode Base64 -// - Link payloads: fetch from HTTP with exponential backoff -// - Deserialize all payloads appropriately -// -// Render mixed content in chat interface -// Support bidirectional reply with claim-check delivery confirmation -``` - -**Python/Micropython (Sender/Receiver):** -```python -# Build chat message with mixed payloads: -# - Text: direct transport (Base64) -# - Small images: direct transport (Base64) -# - Large images: link transport (HTTP URL) -# - Audio/video: link transport (HTTP URL) -# - Tables: direct or link depending on size -# - Files: link transport (HTTP URL) -# -# Each payload uses appropriate transport strategy: -# - Size < 1MB → direct (NATS + Base64) -# - Size >= 1MB → link (HTTP upload + NATS URL) -# -# Include claim-check metadata for delivery tracking -# Support bidirectional messaging with replyTo fields -``` - -**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components across all platforms. +**Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components. **Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads. diff --git a/docs/implementation.md b/docs/implementation.md index 304877b..9a14090 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -2,22 +2,17 @@ ## Overview -This document describes the implementation of the high-performance, bi-directional data bridge between **Julia**, **JavaScript**, and **Python/Micropython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. +This document describes the implementation of the high-performance, bi-directional data bridge for **Julia** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. -The system enables seamless communication across all three platforms: -- **Julia ↔ JavaScript** bi-directional messaging -- **JavaScript ↔ Python/Micropython** bi-directional messaging -- **Julia ↔ Python/Micropython** bi-directional messaging (via JSON serialization) +The system enables seamless communication for Julia applications. ### Implementation Files -NATSBridge is implemented in three languages, each providing the same API: +NATSBridge is implemented in Julia: | Language | Implementation File | Description | |----------|---------------------|-------------| | **Julia** | [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Full Julia implementation with Arrow IPC support | -| **JavaScript** | [`src/NATSBridge.js`](../src/NATSBridge.js) | JavaScript implementation for Node.js and browsers | -| **Python/Micropython** | [`src/nats_bridge.py`](../src/nats_bridge.py) | Python implementation for desktop and microcontrollers | ### File Server Handler Architecture @@ -117,64 +112,9 @@ env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff, max_ret # env is a dictionary containing envelope metadata and payloads field ``` -## Cross-Platform Interoperability - -NATSBridge is designed for seamless communication between Julia, JavaScript, and Python/Micropython applications. All three implementations share the same interface and data format, ensuring compatibility across platforms. - -### Platform-Specific Features - -| Feature | Julia | JavaScript | Python/Micropython | -|---------|-------|------------|-------------------| -| Direct NATS transport | ✅ | ✅ | ✅ | -| HTTP file server (Claim-Check) | ✅ | ✅ | ✅ | -| Arrow IPC tables | ✅ | ✅ | ✅ | -| Base64 encoding | ✅ | ✅ | ✅ | -| Exponential backoff | ✅ | ✅ | ✅ | -| Correlation ID tracking | ✅ | ✅ | ✅ | -| Reply-to support | ✅ | ✅ | ✅ | - -### Data Type Mapping - -| Type | Julia | JavaScript | Python/Micropython | -|------|-------|------------|-------------------| -| `text` | `String` | `String` | `str` | -| `dictionary` | `Dict` | `Object` | `dict` | -| `table` | `DataFrame` | `Array` | `DataFrame` / `list` | -| `image` | `Vector{UInt8}` | `ArrayBuffer/Uint8Array` | `bytes` | -| `audio` | `Vector{UInt8}` | `ArrayBuffer/Uint8Array` | `bytes` | -| `video` | `Vector{UInt8}` | `ArrayBuffer/Uint8Array` | `bytes` | -| `binary` | `Vector{UInt8}` | `ArrayBuffer/Uint8Array` | `bytes` | - -### Example: Julia ↔ Python ↔ JavaScript - -```julia -# Julia sender - smartsend returns (env, env_json_str) -using NATSBridge -data = [("message", "Hello from Julia!", "text")] -env, env_json_str = smartsend("/cross_platform", data, broker_url="nats://localhost:4222") -# env: msg_envelope_v1 with all metadata and payloads -# env_json_str: JSON string for publishing -``` - -```javascript -// JavaScript receiver -const { smartreceive } = require('./src/NATSBridge'); -const env = await smartreceive(msg); -// env.payloads[0].data === "Hello from Julia!" -``` - -```python -# Python sender -from nats_bridge import smartsend -data = [("response", "Hello from Python!", "text")] -smartsend("/cross_platform", data, broker_url="nats://localhost:4222") -``` - -All three platforms can communicate seamlessly using the same NATS subjects and data format. - ## Architecture -All three implementations (Julia, JavaScript, Python/Micropython) follow the same Claim-Check pattern: +The Julia implementation follows the Claim-Check pattern: ``` ┌─────────────────────────────────────────────────────────────────────────┐ @@ -227,24 +167,6 @@ The Julia implementation provides: - **[`smartsend()`](src/NATSBridge.jl)**: Handles transport selection based on payload size - **[`smartreceive()`](src/NATSBridge.jl)**: Handles both direct and link transport -### JavaScript Module: [`src/NATSBridge.js`](../src/NATSBridge.js) - -The JavaScript implementation provides: - -- **`MessageEnvelope` class**: For the unified JSON envelope -- **`MessagePayload` class**: For individual payload representation -- **[`smartsend()`](src/NATSBridge.js)**: Handles transport selection based on payload size -- **[`smartreceive()`](src/NATSBridge.js)**: Handles both direct and link transport - -### Python/Micropython Module: [`src/nats_bridge.py`](../src/nats_bridge.py) - -The Python/Micropython implementation provides: - -- **`MessageEnvelope` class**: For the unified JSON envelope -- **`MessagePayload` class**: For individual payload representation -- **[`smartsend()`](src/nats_bridge.py)**: Handles transport selection based on payload size -- **[`smartreceive()`](src/nats_bridge.py)**: Handles both direct and link transport - ## Installation ### Julia Dependencies @@ -259,29 +181,6 @@ Pkg.add("UUIDs") Pkg.add("Dates") ``` -### JavaScript Dependencies - -```bash -npm install nats.js apache-arrow uuid base64-url -``` - -### Python/Micropython Dependencies - -1. Copy [`src/nats_bridge.py`](../src/nats_bridge.py) to your device -2. Ensure you have the following dependencies: - - **For Python (desktop):** - ```bash - pip install nats-py - ``` - - **For Micropython:** - - `urequests` for HTTP requests - - `base64` for base64 encoding (built-in) - - `json` for JSON handling (built-in) - - `socket` for networking (built-in) - - `uuid` for UUID generation (built-in) - ## Usage Tutorial ### Step 1: Start NATS Server @@ -304,48 +203,21 @@ python3 -m http.server 8080 --directory /tmp/fileserver ### Step 3: Run Test Scenarios ```bash -# Scenario 1: Command & Control (JavaScript sender) -node test/scenario1_command_control.js +# Scenario 1: Command & Control +julia test/scenario1_command_control.jl -# Scenario 2: Large Arrow Table (JavaScript sender) -node test/scenario2_large_table.js +# Scenario 2: Large Arrow Table +julia test/scenario2_large_table.jl # Scenario 3: Julia-to-Julia communication -# Run both Julia and JavaScript versions julia test/scenario3_julia_to_julia.jl -node test/scenario3_julia_to_julia.js ``` -## API Consistency Across Languages - -**High-Level API (Consistent Across All Languages):** -- `smartsend(subject, data, ...)` - Main publishing function -- `smartreceive(msg, ...)` - Main receiving function -- Message envelope structure (`msg_envelope_v1` / `MessageEnvelope`) -- Payload structure (`msg_payload_v1` / `MessagePayload`) -- Transport strategy (direct vs link based on size threshold) -- Supported payload types: text, dictionary, table, image, audio, video, binary - -**Low-Level Native Functions (Language-Specific Conventions):** -- Julia: `NATS.connect()`, `publish_message()`, function overloading -- JavaScript: `nats.js` client, native async/await patterns -- Python: `nats-python` client, native async/await patterns - -**Connection Reuse Pattern - Key Differences:** -- **Julia:** Uses `NATS_connection` keyword parameter with function overloading for automatic connection management -- **JavaScript/Python:** Achieved by creating NATS client outside the function and reusing it in custom handlers or custom publish implementations - -**Why the Difference?** -- Julia supports function overloading and keyword arguments, allowing `NATS_connection` to be passed as an optional parameter -- JavaScript/Python use a simpler `is_publish` option to control automatic publishing -- `is_publish` is simply a switch: when `true`, publish automatically; when `false`, return `(env, env_json_str)` without publishing -- For connection reuse in JavaScript/Python, create a NATS client once and reuse it in your custom `fileserver_upload_handler` or custom publish logic - ## Usage ### Scenario 1: Command & Control (Small Dictionary) -**Focus:** Sending small dictionary configurations across platforms. This is the simplest use case for command and control scenarios. +**Focus:** Sending small dictionary configurations. This is the simplest use case for command and control scenarios. **Julia (Sender/Receiver):** ```julia @@ -386,98 +258,11 @@ NATS.close(conn) **Use Case:** High-frequency publishing scenarios where connection reuse provides performance benefits by avoiding the overhead of establishing a new NATS connection for each message. -**JavaScript (Sender/Receiver):** -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Create small dictionary config -// Send via smartsend with type="dictionary" -const config = { - step_size: 0.01, - iterations: 1000, - threshold: 0.5 -}; - -// Use is_publish option to control automatic publishing -await smartsend("control", [ - { dataname: "config", data: config, type: "dictionary" } -], { - is_publish: true // Automatically publish to NATS -}); -``` - -**Connection Reuse in JavaScript:** -To achieve connection reuse in JavaScript, create a NATS client outside the function and use it in a custom `fileserver_upload_handler` or custom publish implementation: - -```javascript -const { connect } = require('nats'); -const { smartsend } = require('./src/NATSBridge'); - -// Create connection once -const nc = await connect({ servers: ['nats://localhost:4222'] }); - -// Send multiple messages using the same connection -for (let i = 0; i < 100; i++) { - const config = { iteration: i, data: Math.random() }; - - // Option 1: Use is_publish=false and publish manually with your connection - const { env, env_json_str } = await smartsend("control", [ - { dataname: "config", data: config, type: "dictionary" } - ], { is_publish: false }); - - // Publish with your existing connection - await nc.publish("control", env_json_str); -} - -// Close connection when done -await nc.close(); -``` - -**Python/Micropython (Sender/Receiver):** -```python -from nats_bridge import smartsend - -# Create small dictionary config -# Send via smartsend with type="dictionary" -config = { - "step_size": 0.01, - "iterations": 1000, - "threshold": 0.5 -} - -# Use is_publish parameter to control automatic publishing -smartsend("control", [("config", config, "dictionary")], is_publish=True) -``` - -**Connection Reuse in Python:** -To achieve connection reuse in Python, create a NATS client outside the function and use it in a custom `fileserver_upload_handler` or custom publish implementation: - -```python -from nats_bridge import smartsend -import nats - -# Create connection once -nc = await nats.connect("nats://localhost:4222") - -# Send multiple messages using the same connection -for i in range(100): - config = {"iteration": i, "data": random.random()} - - # Option 1: Use is_publish=False and publish manually with your connection - env, env_json_str = smartsend("control", [("config", config, "dictionary")], is_publish=False) - - # Publish with your existing connection - await nc.publish("control", env_json_str) - -# Close connection when done -await nc.close() -``` - ### Basic Multi-Payload Example -#### Python/Micropython (Sender) -```python -from nats_bridge import smartsend +#### Julia (Sender) +```julia +using NATSBridge # Send multiple payloads in one message (type is required per payload) smartsend( @@ -491,71 +276,15 @@ smartsend( smartsend("/test", [("single_data", mydata, "dictionary")], broker_url="nats://localhost:4222") ``` -#### Python/Micropython (Receiver) -```python -from nats_bridge import smartreceive +#### Julia (Receiver) +```julia +using NATSBridge # Receive returns a dictionary with envelope metadata and payloads field env = smartreceive(msg) # env["payloads"] = [(dataname1, data1, "dictionary"), (dataname2, data2, "table"), ...] ``` -#### JavaScript (Sender) -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Single payload wrapped in a list -const config = [{ - dataname: "config", - data: { step_size: 0.01, iterations: 1000 }, - type: "dictionary" -}]; - -await smartsend("control", config, { - correlationId: "unique-id" -}); - -// Multiple payloads -const configs = [ - { - dataname: "config1", - data: { step_size: 0.01 }, - type: "dictionary" - }, - { - dataname: "config2", - data: { iterations: 1000 }, - type: "dictionary" - } -]; - -await smartsend("control", configs); -``` - -#### JavaScript (Receiver) -```javascript -const { smartreceive } = require('./src/NATSBridge'); - -// Subscribe to messages -const nc = await connect({ servers: ['nats://localhost:4222'] }); -const sub = nc.subscribe("control"); - -for await (const msg of sub) { - const env = await smartreceive(msg); - - // Process the payloads from the envelope - for (const payload of env.payloads) { - const { dataname, data, type } = payload; - console.log(`Received ${dataname} of type ${type}`); - console.log(`Data: ${JSON.stringify(data)}`); - } - - // Also access envelope metadata - console.log(`Correlation ID: ${env.correlation_id}`); - console.log(`Message ID: ${env.msg_id}`); -} -``` - ### Scenario 2: Deep Dive Analysis (Large Arrow Table) #### Julia (Sender) @@ -689,91 +418,25 @@ env, env_json_str = smartsend( **API Consistency Note:** - **Julia:** Uses `NATS_connection` keyword parameter with function overloading for automatic connection management -- **JavaScript/Python:** Use `is_publish` option and achieve connection reuse by creating NATS client outside the function and reusing it in custom handlers or custom publish implementations - -#### JavaScript (Receiver) -```javascript -const { smartreceive } = require('./src/NATSBridge'); - -const env = await smartreceive(msg); - -// Use table data from the payloads field -// Note: Tables are sent as arrays of objects in JavaScript -const table = env.payloads; -``` ### Scenario 3: Live Binary Processing -#### Python/Micropython (Sender) -```python -from nats_bridge import smartsend +**Julia (Sender/Receiver):** +```julia +using NATSBridge # Binary data wrapped in list with type smartsend( "binary_input", [("audio_chunk", binary_buffer, "binary")], broker_url="nats://localhost:4222", - metadata={"sample_rate": 44100, "channels": 1} + metadata=["sample_rate" => 44100, "channels" => 1] ) ``` -#### JavaScript (Sender) -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Binary data wrapped in a list -const binaryData = [{ - dataname: "audio_chunk", - data: binaryBuffer, // ArrayBuffer or Uint8Array - type: "binary" -}]; - -await smartsend("binary_input", binaryData, { - metadata: { - sample_rate: 44100, - channels: 1 - } -}); -``` - -#### Python/Micropython (Receiver) -```python -from nats_bridge import smartreceive - -# Receive binary data -def process_binary(msg): - env = smartreceive(msg) - - # Process the binary data from env.payloads - for dataname, data, type in env["payloads"]: - if type == "binary": - # data is bytes - print(f"Received binary data: {dataname}, size: {len(data)}") - # Perform FFT or AI transcription here -``` - -#### JavaScript (Receiver) -```javascript -const { smartreceive } = require('./src/NATSBridge'); - -// Receive binary data -function process_binary(msg) { - const env = await smartreceive(msg); - - // Process the binary data from env.payloads - for (const payload of env.payloads) { - if (payload.type === "binary") { - // data is an ArrayBuffer or Uint8Array - console.log(`Received binary data: ${payload.dataname}, size: ${payload.data.length}`); - // Perform FFT or AI transcription here - } - } -} -``` - ### Scenario 4: Catch-Up (JetStream) -#### Julia (Producer) +**Julia (Producer/Consumer):** ```julia using NATSBridge @@ -789,93 +452,11 @@ function publish_health_status(broker_url) end ``` -#### JavaScript (Consumer) -```javascript -const { connect } = require('nats'); -const { smartreceive } = require('./src/NATSBridge'); - -const nc = await connect({ servers: ['nats://localhost:4222'] }); -const js = nc.jetstream(); - -// Request replay from last 10 minutes -const consumer = await js.pullSubscribe("health", { - durable_name: "catchup", - max_batch: 100, - max_ack_wait: 30000 -}); - -// Process historical and real-time messages -for await (const msg of consumer) { - const env = await smartreceive(msg); - // env.payloads contains the list of payloads - // Each payload has: dataname, data, type - msg.ack(); -} -``` - -### Scenario 4: Micropython Device Control - -**Focus:** Sending configuration to a Micropython device over NATS. This demonstrates the lightweight nature of the Python implementation suitable for microcontrollers. - -**Python/Micropython (Receiver/Device):** -```python -from nats_bridge import smartsend, smartreceive -import json - -# Device configuration handler -def handle_device_config(msg): - env = smartreceive(msg) - - # Process configuration from payloads - for dataname, data, payload_type in env["payloads"]: - if payload_type == "dictionary": - print(f"Received configuration: {data}") - # Apply configuration to device - if "wifi_ssid" in data: - wifi_ssid = data["wifi_ssid"] - wifi_password = data["wifi_password"] - update_wifi_config(wifi_ssid, wifi_password) - - # Send confirmation back - config = { - "status": "configured", - "wifi_ssid": "MyNetwork", - "ip": get_device_ip() - } - smartsend( - "device/response", - [("config", config, "dictionary")], - broker_url="nats://localhost:4222", - reply_to=env.get("reply_to") - ) -``` - -**JavaScript (Sender/Controller):** -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Send configuration to Micropython device -await smartsend("device/config", [ - { - dataname: "config", - data: { - wifi_ssid: "MyNetwork", - wifi_password: "password123", - update_interval: 60, - temperature_threshold: 30.0 - }, - type: "dictionary" - } -]); -``` - -**Use Case:** A controller sends WiFi and operational configuration to a Micropython device (e.g., ESP32). The device receives the configuration, applies it, and sends back a confirmation with its current status. - ### Scenario 5: Selection (Low Bandwidth) -**Focus:** Small Arrow tables, Julia to JavaScript. The Action: Julia wants to send a small DataFrame to show on a JavaScript dashboard for the user to choose. +**Focus:** Small Arrow tables. The Action: Julia wants to send a small DataFrame to show on a receiving application for the user to choose. -**Julia (Sender):** +**Julia (Sender/Receiver):** ```julia using NATSBridge using DataFrames @@ -903,27 +484,7 @@ env, env_json_str = smartsend( # env_json_str: JSON string for publishing ``` -**JavaScript (Receiver):** -```javascript -const { smartreceive, smartsend } = require('./src/NATSBridge'); - -// Receive NATS message with direct transport -const env = await smartreceive(msg); - -// Decode Base64 payload (for direct transport) -// For tables, data is in env.payloads -const table = env.payloads; // Array of objects - -// User makes selection -const selection = uiComponent.getSelectedOption(); - -// Send selection back to Julia -await smartsend("dashboard.response", [ - { dataname: "selected_option", data: selection, type: "dictionary" } -]); -``` - -**Use Case:** Julia server generates a list of available options (e.g., file selections, configuration presets) as a small DataFrame and sends to JavaScript dashboard for user selection. The selection is then sent back to Julia for processing. +**Use Case:** Julia server generates a list of available options (e.g., file selections, configuration presets) as a small DataFrame and sends to a receiving application for user selection. The selection is then sent back to Julia for processing. ### Scenario 6: Chat System @@ -968,46 +529,6 @@ env, env_json_str = smartsend( # env_json_str: JSON string for publishing ``` -**JavaScript (Sender/Receiver):** -```javascript -const { smartsend, smartreceive } = require('./src/NATSBridge'); - -// Build chat message with mixed content: -// - User input text: direct transport -// - Selected image: check size, use appropriate transport -// - Audio recording: link transport for large files -// - File attachment: link transport -// -// Parse received message: -// - Direct payloads: decode Base64 -// - Link payloads: fetch from HTTP with exponential backoff -// - Deserialize all payloads appropriately -// -// Render mixed content in chat interface -// Support bidirectional reply with claim-check delivery confirmation - -// Example: Send chat with mixed content -const message = [ - { - dataname: "text", - data: "Hello from JavaScript!", - type: "text" - }, - { - dataname: "image", - data: selectedImageBuffer, // Small image (ArrayBuffer or Uint8Array) - type: "image" - }, - { - dataname: "audio", - data: audioUrl, // Large audio, link transport - type: "audio" - } -]; - -await smartsend("chat.room123", message); -``` - **Use Case:** Full-featured chat system supporting rich media. User can send text, small images directly, or upload large files that get uploaded to HTTP server and referenced via URLs. Claim-check pattern ensures reliable delivery tracking for all message components. **Implementation Note:** The `smartreceive` function iterates through all payloads in the envelope and processes each according to its transport type. See the standard API format in Section 1: `msg_envelope_v1` supports `Vector{msg_payload_v1}` for multiple payloads. @@ -1072,7 +593,6 @@ await smartsend("chat.room123", message); ### Exponential Backoff - Maximum retry count: 5 - Base delay: 100ms, max delay: 5000ms -- Implemented in all three implementations (Julia, JavaScript, Python/Micropython) ### Correlation ID Logging - Log correlation_id at every stage @@ -1081,38 +601,7 @@ await smartsend("chat.room123", message); ## Testing -Run the test scripts for each platform: - -### Python/Micropython Tests - -```bash -# Basic functionality test -python test/test_micropython_basic.py -``` - -### JavaScript Tests - -```bash -# Text message exchange -node test/test_js_to_js_text_sender.js -node test/test_js_to_js_text_receiver.js - -# Dictionary exchange -node test/test_js_to_js_dict_sender.js -node test/test_js_to_js_dict_receiver.js - -# File transfer (direct transport) -node test/test_js_to_js_file_sender.js -node test/test_js_to_js_file_receiver.js - -# Mixed payload types -node test/test_js_to_js_mix_payloads_sender.js -node test/test_js_to_js_mix_payloads_receiver.js - -# Table (Arrow IPC) exchange -node test/test_js_to_js_table_sender.js -node test/test_js_to_js_table_receiver.js -``` +Run the test scripts for Julia: ### Julia Tests @@ -1138,41 +627,22 @@ julia test/test_julia_to_julia_table_sender.jl julia test/test_julia_to_julia_table_receiver.jl ``` -### Cross-Platform Tests - -```bash -# Julia ↔ JavaScript communication -julia test/test_julia_to_julia_text_sender.jl -node test/test_js_to_js_text_receiver.js - -# Python ↔ JavaScript communication -python test/test_micropython_basic.py -node test/test_js_to_js_text_receiver.js -``` - ## Troubleshooting ### Common Issues 1. **NATS Connection Failed** - - **Julia/JavaScript/Python**: Ensure NATS server is running - - **Python/Micropython**: Check `nats_url` parameter and network connectivity + - Ensure NATS server is running 2. **HTTP Upload Failed** - Ensure file server is running - Check `fileserver_url` configuration - Verify upload permissions - - **Micropython**: Ensure `urequests` is available and network is connected 3. **Arrow IPC Deserialization Error** - Ensure data is properly serialized to Arrow format - Check Arrow version compatibility -4. **Python/Micropython Specific Issues** - - **Import Error**: Ensure `nats_bridge.py` is in the correct path - - **Memory Error (Micropython)**: Reduce payload size or use link transport for large payloads - - **Unicode Error**: Ensure proper encoding when sending text data - ## License MIT \ No newline at end of file diff --git a/examples/tutorial.md b/examples/tutorial.md index 6f58eab..474f867 100644 --- a/examples/tutorial.md +++ b/examples/tutorial.md @@ -1,6 +1,6 @@ # NATSBridge Tutorial -A step-by-step guide to get started with NATSBridge - a high-performance, bi-directional data bridge for **Julia**, **JavaScript**, and **Python/Micropython**. +A step-by-step guide to get started with NATSBridge - a high-performance, bi-directional data bridge for **Julia**. ## Table of Contents @@ -10,13 +10,12 @@ A step-by-step guide to get started with NATSBridge - a high-performance, bi-dir 4. [Quick Start](#quick-start) 5. [Basic Examples](#basic-examples) 6. [Advanced Usage](#advanced-usage) -7. [Cross-Platform Communication](#cross-platform-communication) --- ## Overview -NATSBridge enables seamless communication between Julia, JavaScript, and Python/Micropython applications through NATS, with automatic transport selection based on payload size: +NATSBridge enables seamless communication for Julia applications through NATS, with automatic transport selection based on payload size: - **Direct Transport**: Payloads < 1MB are sent directly via NATS (Base64 encoded) - **Link Transport**: Payloads >= 1MB are uploaded to an HTTP file server and referenced via URL @@ -41,7 +40,7 @@ Before you begin, ensure you have: 1. **NATS Server** running (or accessible) 2. **HTTP File Server** (optional, for large payloads > 1MB) -3. **One of the supported platforms**: Julia, JavaScript (Node.js), or Python/Micropython +3. **Julia** with required packages --- @@ -59,27 +58,6 @@ Pkg.add("UUIDs") Pkg.add("Dates") ``` -### JavaScript - -```bash -npm install nats.js apache-arrow uuid base64-url -``` - -### Python/Micropython - -1. Copy `src/nats_bridge.py` to your device -2. Install dependencies: - -**For Python (desktop):** -```bash -pip install nats-py -``` - -**For Micropython:** -- `urequests` for HTTP requests -- `base64` for base64 encoding (built-in) -- `json` for JSON handling (built-in) - --- ## Quick Start @@ -96,48 +74,12 @@ docker run -p 4222:4222 nats:latest # Create a directory for file uploads mkdir -p /tmp/fileserver -# Use Python's built-in server +# Use any HTTP server that supports POST for file uploads python3 -m http.server 8080 --directory /tmp/fileserver ``` ### Step 3: Send Your First Message -#### Python/Micropython - -```python -from nats_bridge import smartsend - -# Send a text message (is_publish=True by default) -data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") -print("Message sent!") - -# Or use is_publish=False to get envelope and JSON without publishing -env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=False) -# env: MessageEnvelope object -# env_json_str: JSON string for publishing to NATS -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Send a text message (isPublish=true by default) -await smartsend("/chat/room1", [ - { dataname: "message", data: "Hello World", type: "text" } -], { brokerUrl: "nats://localhost:4222" }); - -console.log("Message sent!"); - -// Or use isPublish=false to get envelope and JSON without publishing -const { env, env_json_str } = await smartsend("/chat/room1", [ - { dataname: "message", data: "Hello World", type: "text" } -], { brokerUrl: "nats://localhost:4222", isPublish: false }); -// env: MessageEnvelope object -// env_json_str: JSON string for publishing to NATS -``` - #### Julia ```julia @@ -149,33 +91,15 @@ env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost: # env: msg_envelope_v1 object with all metadata and payloads # env_json_str: JSON string representation of the envelope for publishing println("Message sent!") + +# Or use is_publish=false to get envelope and JSON without publishing +env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=false) +# env: msg_envelope_v1 object +# env_json_str: JSON string for publishing to NATS ``` ### Step 4: Receive Messages -#### Python/Micropython - -```python -from nats_bridge import smartreceive - -# Receive and process message -env = smartreceive(msg) -for dataname, data, type in env["payloads"]: - print(f"Received {dataname}: {data}") -``` - -#### JavaScript - -```javascript -const { smartreceive } = require('./src/NATSBridge'); - -// Receive and process message -const env = await smartreceive(msg); -for (const payload of env.payloads) { - console.log(`Received ${payload.dataname}: ${payload.data}`); -} -``` - #### Julia ```julia @@ -194,39 +118,6 @@ end ### Example 1: Sending a Dictionary -#### Python/Micropython - -```python -from nats_bridge import smartsend - -# Create configuration dictionary -config = { - "wifi_ssid": "MyNetwork", - "wifi_password": "password123", - "update_interval": 60 -} - -# Send as dictionary type -data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222") -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const config = { - wifi_ssid: "MyNetwork", - wifi_password: "password123", - update_interval: 60 -}; - -const { env, env_json_str } = await smartsend("/device/config", [ - { dataname: "config", data: config, type: "dictionary" } -], { brokerUrl: "nats://localhost:4222" }); -``` - #### Julia ```julia @@ -244,34 +135,6 @@ env, env_json_str = smartsend("/device/config", data, broker_url="nats://localho ### Example 2: Sending Binary Data (Image) -#### Python/Micropython - -```python -from nats_bridge import smartsend - -# Read image file -with open("image.png", "rb") as f: - image_data = f.read() - -# Send as binary type -data = [("user_image", image_data, "binary")] -env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222") -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Read image file (Node.js) -const fs = require('fs'); -const image_data = fs.readFileSync('image.png'); - -const { env, env_json_str } = await smartsend("/chat/image", [ - { dataname: "user_image", data: image_data, type: "binary" } -], { brokerUrl: "nats://localhost:4222" }); -``` - #### Julia ```julia @@ -286,13 +149,13 @@ env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost: ### Example 3: Request-Response Pattern -#### Python/Micropython (Requester) +#### Julia (Requester) -```python -from nats_bridge import smartsend +```julia +using NATSBridge # Send command with reply-to -data = [("command", {"action": "read_sensor"}, "dictionary")] +data = [("command", Dict("action" => "read_sensor"), "dictionary")] env, env_json_str = smartsend( "/device/command", data, @@ -300,44 +163,43 @@ env, env_json_str = smartsend( reply_to="/device/response", reply_to_msg_id="cmd-001" ) -# env: MessageEnvelope object +# env: msg_envelope_v1 object # env_json_str: JSON string for publishing to NATS ``` -#### JavaScript (Responder) +#### Julia (Responder) -```javascript -const { smartreceive, smartsend } = require('./src/NATSBridge'); +```julia +using NATS, NATSBridge -// Subscribe to command topic -const sub = nc.subscribe("/device/command"); +# Configuration +const SUBJECT = "/device/command" +const NATS_URL = "nats://localhost:4222" -for await (const msg of sub) { - const env = await smartreceive(msg); +function test_responder() + conn = NATS.connect(NATS_URL) + NATS.subscribe(conn, SUBJECT) do msg + env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) + + # Extract reply_to from the envelope metadata + reply_to = env["reply_to"] + + for (dataname, data, type) in env["payloads"] + if dataname == "command" && data["action"] == "read_sensor" + response = Dict("sensor_id" => "sensor-001", "value" => 42.5) + # Send response to the reply_to subject from the request + if !isempty(reply_to) + smartsend(reply_to, [("data", response, "dictionary")]) + end + end + end + end - // Process command - for (const payload of env.payloads) { - if (payload.dataname === "command") { - const command = payload.data; - - if (command.action === "read_sensor") { - // Read sensor and send response - const response = { - sensor_id: "sensor-001", - value: 42.5, - timestamp: new Date().toISOString() - }; - - await smartsend("/device/response", [ - { dataname: "sensor_data", data: response, type: "dictionary" } - ], { - reply_to: env.replyTo, - reply_to_msg_id: env.msgId - }); - } - } - } -} + sleep(120) + NATS.drain(conn) +end + +test_responder() ``` --- @@ -348,47 +210,6 @@ for await (const msg of sub) { For payloads larger than 1MB, NATSBridge automatically uses the file server: -#### Python/Micropython - -```python -from nats_bridge import smartsend -import os - -# Create large data (> 1MB) -large_data = os.urandom(2_000_000) # 2MB of random data - -# Send with file server URL -env, env_json_str = smartsend( - "/data/large", - [("large_file", large_data, "binary")], - broker_url="nats://localhost:4222", - fileserver_url="http://localhost:8080", - size_threshold=1_000_000 -) - -# The envelope will contain the download URL -print(f"File uploaded to: {env.payloads[0].data}") -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -// Create large data (> 1MB) -const largeData = new ArrayBuffer(2_000_000); -const view = new Uint8Array(largeData); -view.fill(42); // Fill with some data - -const { env, env_json_str } = await smartsend("/data/large", [ - { dataname: "large_file", data: largeData, type: "binary" } -], { - brokerUrl: "nats://localhost:4222", - fileserverUrl: "http://localhost:8080", - sizeThreshold: 1_000_000 -}); -``` - #### Julia ```julia @@ -412,45 +233,6 @@ println("File uploaded to: $(env.payloads[1].data)") NATSBridge supports sending multiple payloads with different types in a single message: -#### Python/Micropython - -```python -from nats_bridge import smartsend - -# Read image file -with open("avatar.png", "rb") as f: - image_data = f.read() - -# Send mixed content -data = [ - ("message_text", "Hello with image!", "text"), - ("user_avatar", image_data, "image") -] - -env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222") -``` - -#### JavaScript - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const fs = require('fs'); - -const { env, env_json_str } = await smartsend("/chat/mixed", [ - { - dataname: "message_text", - data: "Hello with image!", - type: "text" - }, - { - dataname: "user_avatar", - data: fs.readFileSync("avatar.png"), - type: "image" - } -], { brokerUrl: "nats://localhost:4222" }); -``` - #### Julia ```julia @@ -470,24 +252,6 @@ env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost: For tabular data, NATSBridge uses Apache Arrow IPC format: -#### Python/Micropython - -```python -from nats_bridge import smartsend -import pandas as pd - -# Create DataFrame -df = pd.DataFrame({ - "id": [1, 2, 3], - "name": ["Alice", "Bob", "Charlie"], - "score": [95, 88, 92] -}) - -# Send as table type -data = [("students", df, "table")] -env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222") -``` - #### Julia ```julia @@ -507,92 +271,10 @@ env, env_json_str = smartsend("/data/students", data, broker_url="nats://localho --- -## Cross-Platform Communication - -NATSBridge enables seamless communication between different platforms: - -### Julia ↔ JavaScript - -#### Julia Sender - -```julia -using NATSBridge - -# Send dictionary from Julia to JavaScript -config = Dict("step_size" => 0.01, "iterations" => 1000) -data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/analysis/config", data, broker_url="nats://localhost:4222") -``` - -#### JavaScript Receiver - -```javascript -const { smartreceive } = require('./src/NATSBridge'); - -// Receive dictionary from Julia -const env = await smartreceive(msg); -for (const payload of env.payloads) { - if (payload.type === "dictionary") { - console.log("Received config:", payload.data); - // payload.data = { step_size: 0.01, iterations: 1000 } - } -} -``` - -### JavaScript ↔ Python - -#### JavaScript Sender - -```javascript -const { smartsend } = require('./src/NATSBridge'); - -const { env, env_json_str } = await smartsend("/data/transfer", [ - { dataname: "message", data: "Hello from JS!", type: "text" } -], { brokerUrl: "nats://localhost:4222" }); -``` - -#### Python Receiver - -```python -from nats_bridge import smartreceive - -env = smartreceive(msg) -for dataname, data, type in env["payloads"]: - if type == "text": - print(f"Received from JS: {data}") -``` - -### Python ↔ Julia - -#### Python Sender - -```python -from nats_bridge import smartsend - -data = [("message", "Hello from Python!", "text")] -env, env_json_str = smartsend("/chat/python", data, broker_url="nats://localhost:4222") -``` - -#### Julia Receiver - -```julia -using NATSBridge - -env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff) -for (dataname, data, type) in env["payloads"] - if type == "text" - println("Received from Python: $data") - end -end -``` - ---- - ## Next Steps 1. **Explore the test directory** for more examples 2. **Check the documentation** for advanced configuration options -3. **Join the community** to share your use cases --- @@ -613,7 +295,7 @@ end ### Serialization Errors - Verify data type matches the specified type -- Check that binary data is in the correct format (bytes/Vector{UInt8}) +- Check that binary data is in the correct format (Vector{UInt8}) --- diff --git a/examples/walkthrough.md b/examples/walkthrough.md index 8f51475..e74bca0 100644 --- a/examples/walkthrough.md +++ b/examples/walkthrough.md @@ -9,10 +9,8 @@ A comprehensive guide to building real-world applications with NATSBridge. 3. [Building a Chat Application](#building-a-chat-application) 4. [Building a File Transfer System](#building-a-file-transfer-system) 5. [Building a Streaming Data Pipeline](#building-a-streaming-data-pipeline) -6. [Building a Micropython IoT Device](#building-a-micropython-iot-device) -7. [Building a Cross-Platform Dashboard](#building-a-cross-platform-dashboard) -8. [Performance Optimization](#performance-optimimization) -9. [Best Practices](#best-practices) +6. [Performance Optimization](#performance-optimimization) +7. [Best Practices](#best-practices) --- @@ -23,8 +21,6 @@ This walkthrough will guide you through building several real-world applications - Chat applications with rich media support - File transfer systems with claim-check pattern - Streaming data pipelines -- Micropython IoT devices -- Cross-platform dashboards Each section builds on the previous one, gradually increasing in complexity. @@ -38,22 +34,16 @@ Each section builds on the previous one, gradually increasing in complexity. ┌─────────────────────────────────────────────────────────────────┐ │ NATSBridge Architecture │ ├─────────────────────────────────────────────────────────────────┤ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Julia │ │ JavaScript │ │ Python/Micr │ │ -│ │ (NATS.jl) │◄──►│ (nats.js) │◄──►│ opython │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────────────────────────────────────┐ │ -│ │ NATS │ │ -│ │ (Message Broker) │ │ -│ └──────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────┐ │ -│ │ File Server │ │ -│ │ (HTTP Upload) │ │ -│ └──────────────────┘ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ Julia │ │ NATS │ │ +│ │ (NATS.jl) │◄──►│ Server │ │ +│ └──────────────┘ └──────────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────┐ │ +│ │ File Server │ │ +│ │ (HTTP Upload) │ │ +│ └──────────────────────────────────────┘ │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -89,176 +79,118 @@ cat > config.json << 'EOF' EOF ``` -### Step 2: Create the Chat Interface (JavaScript) +### Step 2: Create the Chat Interface (Julia) -```javascript -// src/chat-ui.js -class ChatUI { - constructor() { - this.messages = []; - this.currentRoom = null; - this.setupEventListeners(); - } +```julia +# src/chat_ui.jl +using NATSBridge, NATS + +struct ChatUI + messages::Vector{Dict} + current_room::String +end + +function ChatUI() + ChatUI(Dict[], "") +end + +function send_message(ui::ChatUI, message_input::String, selected_file::Union{Nothing, String}) + data = [] - setupEventListeners() { - document.getElementById('send-btn').addEventListener('click', () => this.sendMessage()); - document.getElementById('file-input').addEventListener('change', (e) => this.handleFileSelect(e)); - } + # Add text message + if !isempty(message_input) + push!(data, ("text", message_input, "text")) + end - async sendMessage() { - const messageInput = document.getElementById('message-input'); - const text = messageInput.value.trim(); - - if (!text && !this.selectedFile) return; - - const data = []; - - // Add text message - if (text) { - data.push({ - dataname: "text", - data: text, - type: "text" - }); - } - - // Add file if selected - if (this.selectedFile) { - const fileData = await this.readFile(this.selectedFile); - data.push({ - dataname: "attachment", - data: fileData, - type: this.getFileType(this.selectedFile.type) - }); - } - - const { env, env_json_str } = await smartsend( - `/chat/${this.currentRoom}`, - data, - { - brokerUrl: window.config.broker_url, - fileserverUrl: window.config.fileserver_url, - sizeThreshold: window.config.size_threshold - } - ); - - messageInput.value = ''; - this.selectedFile = null; - document.getElementById('file-name').textContent = ''; - } + # Add file if selected + if selected_file !== nothing + file_data = read(selected_file) + file_type = get_file_type(selected_file) + push!(data, ("attachment", file_data, file_type)) + end - handleFileSelect(event) { - this.selectedFile = event.target.files[0]; - document.getElementById('file-name').textContent = `Selected: ${this.selectedFile.name}`; - } - - readFile(file) { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(reader.result); - reader.onerror = reject; - reader.readAsArrayBuffer(file); - }); - } - - getFileType(mimeType) { - if (mimeType.startsWith('image/')) return 'image'; - if (mimeType.startsWith('audio/')) return 'audio'; - if (mimeType.startsWith('video/')) return 'video'; - return 'binary'; - } - - addMessage(user, text, attachment = null) { - const messagesContainer = document.getElementById('messages'); - const messageDiv = document.createElement('div'); - messageDiv.className = 'message'; - - let content = `
${user}
`; - content += `
${text}
`; - - if (attachment) { - if (attachment.type === 'image') { - content += ``; - } else { - content += `Download attachment`; - } - } - - messageDiv.innerHTML = content; - messagesContainer.appendChild(messageDiv); - messagesContainer.scrollTop = messagesContainer.scrollHeight; - } -} + return data +end + +function get_file_type(filename::String)::String + if endswith(filename, ".png") || endswith(filename, ".jpg") + return "image" + elseif endswith(filename, ".mp3") || endswith(filename, ".wav") + return "audio" + elseif endswith(filename, ".mp4") || endswith(filename, ".avi") + return "video" + else + return "binary" + end +end + +function add_message(ui::ChatUI, user::String, text::String, attachment::Union{Nothing, Dict}) + push!(ui.messages, Dict( + "user" => user, + "text" => text, + "attachment" => attachment + )) +end ``` ### Step 3: Create the Message Handler -```javascript -// src/chat-handler.js -const { smartreceive } = require('./NATSBridge'); +```julia +# src/chat_handler.jl +using NATSBridge, NATS -class ChatHandler { - constructor(natsConnection) { - this.nats = natsConnection; - this.ui = new ChatUI(); - } +struct ChatHandler + nats::NATS.Connection + ui::ChatUI +end + +function ChatHandler(nats_connection::NATS.Connection) + ChatHandler(nats_connection, ChatUI()) +end + +function start(handler::ChatHandler) + # Subscribe to chat rooms + rooms = ["general", "tech", "random"] - async start() { - // Subscribe to chat rooms - const rooms = ['general', 'tech', 'random']; - - for (const room of rooms) { - await this.nats.subscribe(`/chat/${room}`, async (msg) => { - await this.handleMessage(msg); - }); - } - - console.log('Chat handler started'); - } + for room in rooms + NATS.subscribe(handler.nats, "/chat/$room") do msg + handle_message(handler, msg) + end + end - async handleMessage(msg) { - const env = await smartreceive(msg, { - fileserverDownloadHandler: this.downloadFile.bind(this) - }); - - // Extract sender info from envelope - const sender = env.senderName || 'Anonymous'; - - // Process each payload - for (const payload of env.payloads) { - if (payload.type === 'text') { - this.ui.addMessage(sender, payload.data); - } else if (payload.type === 'image') { - // Convert to data URL for display - const base64 = this.arrayBufferToBase64(payload.data); - this.ui.addMessage(sender, null, { - type: 'image', - data: `data:image/png;base64,${base64}` - }); - } else { - // For other types, use file server URL - this.ui.addMessage(sender, null, { - type: payload.type, - data: payload.data - }); - } - } - } + println("Chat handler started") +end + +function handle_message(handler::ChatHandler, msg::NATS.Msg) + env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff) - downloadFile(url, max_retries, base_delay, max_delay, correlation_id) { - return fetch(url) - .then(response => response.arrayBuffer()); - } + # Extract sender info from envelope + sender = get(env, "sender_name", "Anonymous") - arrayBufferToBase64(buffer) { - const bytes = new Uint8Array(buffer); - let binary = ''; - for (let i = 0; i < bytes.length; i++) { - binary += String.fromCharCode(bytes[i]); - } - return btoa(binary); - } -} + # Process each payload + for (dataname, data, type) in env["payloads"] + if type == "text" + add_message(handler.ui, sender, data, nothing) + elseif type == "image" + # Convert to data URL for display + base64_data = base64encode(data) + attachment = Dict( + "type" => "image", + "data" => "data:image/png;base64,$base64_data" + ) + add_message(handler.ui, sender, "", attachment) + else + # For other types, use file server URL + attachment = Dict("type" => type, "data" => data) + add_message(handler.ui, sender, "", attachment) + end + end +end + +function download_file(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} + # Implement exponential backoff for file server downloads + # Return downloaded data as Vector{UInt8} +end ``` ### Step 4: Run the Application @@ -272,8 +204,8 @@ mkdir -p /tmp/fileserver python3 -m http.server 8080 --directory /tmp/fileserver # Run chat app -node src/chat-ui.js -node src/chat-handler.js +julia src/chat_ui.jl +julia src/chat_handler.jl ``` --- @@ -282,168 +214,148 @@ node src/chat-handler.js Let's build a file transfer system that handles large files efficiently. -### Step 1: File Upload Service +### Step 1: File Upload Service (Julia) -```javascript -// src/file-upload-service.js -const { smartsend } = require('./NATSBridge'); +```julia +# src/file_upload_service.jl +using NATSBridge, HTTP -class FileUploadService { - constructor(brokerUrl, fileserverUrl) { - this.brokerUrl = brokerUrl; - this.fileserverUrl = fileserverUrl; - } - - async uploadFile(filePath, recipient) { - const fs = require('fs'); - const fileData = fs.readFileSync(filePath); - const fileName = require('path').basename(filePath); - - const data = [{ - dataname: fileName, - data: fileData, - type: 'binary' - }]; - - const { env, env_json_str } = await smartsend( - `/files/${recipient}`, - data, - { - brokerUrl: this.brokerUrl, - fileserverUrl: this.fileserverUrl, - sizeThreshold: 1048576 - } - ); - - return env; - } - - async uploadLargeFile(filePath, recipient) { - // For very large files, stream upload - const fs = require('fs'); - const fileSize = fs.statSync(filePath).size; - - if (fileSize > 100 * 1024 * 1024) { // > 100MB - console.log('File too large for direct upload, using streaming...'); - return await this.streamUpload(filePath, recipient); - } - - return await this.uploadFile(filePath, recipient); - } - - async streamUpload(filePath, recipient) { - // Implement streaming upload to file server - // This would require a more sophisticated file server - // For now, we'll use the standard upload - return await this.uploadFile(filePath, recipient); - } -} +struct FileUploadService + broker_url::String + fileserver_url::String +end -module.exports = FileUploadService; +function FileUploadService(broker_url::String, fileserver_url::String) + FileUploadService(broker_url, fileserver_url) +end + +function upload_file(service::FileUploadService, file_path::String, recipient::String)::Dict + file_data = read(file_path) + file_name = basename(file_path) + + data = [("file", file_data, "binary")] + + env, env_json_str = smartsend( + "/files/$recipient", + data, + broker_url=service.broker_url, + fileserver_url=service.fileserver_url + ) + + return env +end + +function upload_large_file(service::FileUploadService, file_path::String, recipient::String)::Dict + file_size = stat(file_path).size + + if file_size > 100 * 1024 * 1024 # > 100MB + println("File too large for direct upload, using streaming...") + return stream_upload(service, file_path, recipient) + end + + return upload_file(service, file_path, recipient) +end + +function stream_upload(service::FileUploadService, file_path::String, recipient::String)::Dict + # Implement streaming upload to file server + # This would require a more sophisticated file server + # For now, we'll use the standard upload + return upload_file(service, file_path, recipient) +end ``` -### Step 2: File Download Service +### Step 2: File Download Service (Julia) -```javascript -// src/file-download-service.js -const { smartreceive } = require('./NATSBridge'); -const fs = require('fs'); +```julia +# src/file_download_service.jl +using NATSBridge -class FileDownloadService { - constructor(natsUrl) { - this.natsUrl = natsUrl; - this.downloads = new Map(); - } +struct FileDownloadService + nats_url::String +end + +function FileDownloadService(nats_url::String) + FileDownloadService(nats_url) +end + +function download_file(service::FileDownloadService, msg::NATS.Msg, sender::String, download_id::String) + # Subscribe to sender's file channel + env = smartreceive(msg, fileserver_download_handler=fetch_from_url) - async downloadFile(sender, downloadId) { - // Subscribe to sender's file channel - const env = await smartreceive(msg, { - fileserverDownloadHandler: this.fetchFromUrl.bind(this) - }); - - // Process each payload - for (const payload of env.payloads) { - if (payload.type === 'binary') { - const filePath = `/downloads/${payload.dataname}`; - fs.writeFileSync(filePath, payload.data); - console.log(`File saved to ${filePath}`); - } - } - } - - async fetchFromUrl(url, max_retries, base_delay, max_delay, correlation_id) { - const response = await fetch(url); - if (!response.ok) { - throw new Error(`Failed to fetch: ${response.status}`); - } - return await response.arrayBuffer(); - } -} + # Process each payload + for (dataname, data, type) in env["payloads"] + if type == "binary" + file_path = "/downloads/$dataname" + write(file_path, data) + println("File saved to $file_path") + end + end +end -module.exports = FileDownloadService; +function fetch_from_url(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} + # Fetch data from URL with exponential backoff + # Return downloaded data as Vector{UInt8} +end ``` -### Step 3: File Transfer CLI +### Step 3: File Transfer CLI (Julia) -```javascript -// src/cli.js -const { smartsend, smartreceive } = require('./NATSBridge'); -const fs = require('fs'); -const readline = require('readline'); +```julia +# src/cli.jl +using NATSBridge, Readlines, FileIO -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}); +function main() + config = JSON3.read(read("config.json", String)) + + println("File Transfer System") + println("====================") + println("1. Upload file") + println("2. Download file") + println("3. List pending downloads") + + print("Enter choice: ") + choice = readline() + + if choice == "1" + upload_file_cli(config) + elseif choice == "2" + download_file_cli(config) + end +end -async function main() { - const config = JSON.parse(fs.readFileSync('config.json', 'utf8')); +function upload_file_cli(config) + print("Enter file path: ") + file_path = readline() - console.log('File Transfer System'); - console.log('===================='); - console.log('1. Upload file'); - console.log('2. Download file'); - console.log('3. List pending downloads'); + print("Enter recipient: ") + recipient = readline() - const choice = await rl.question('Enter choice: '); + file_service = FileUploadService(config.nats_url, config.fileserver_url) - if (choice === '1') { - await uploadFile(config); - } else if (choice === '2') { - await downloadFile(config); - } - - rl.close(); -} + try + env = upload_file(file_service, file_path, recipient) + println("Upload successful!") + println("File ID: $(env["payloads"][1][1])") + catch error + println("Upload failed: $(error)") + end +end -async function uploadFile(config) { - const filePath = await rl.question('Enter file path: '); - const recipient = await rl.question('Enter recipient: '); +function download_file_cli(config) + print("Enter sender: ") + sender = readline() - const fileService = new FileUploadService(config.broker_url, config.fileserver_url); + file_service = FileDownloadService(config.nats_url) - try { - const env = await fileService.uploadFile(filePath, recipient); - console.log('Upload successful!'); - console.log(`File ID: ${env.payloads[0].id}`); - } catch (error) { - console.error('Upload failed:', error.message); - } -} + try + download_file(file_service, sender) + println("Download complete!") + catch error + println("Download failed: $(error)") + end +end -async function downloadFile(config) { - const sender = await rl.question('Enter sender: '); - const fileService = new FileDownloadService(config.nats_url); - - try { - await fileService.downloadFile(sender); - console.log('Download complete!'); - } catch (error) { - console.error('Download failed:', error.message); - } -} - -main(); +main() ``` --- @@ -452,481 +364,175 @@ main(); Let's build a data pipeline that processes streaming data from sensors. -### Step 1: Sensor Data Model +### Step 1: Sensor Data Model (Julia) -```python -# src/sensor_data.py -from dataclasses import dataclass -from datetime import datetime -from typing import List, Dict, Any -import json +```julia +# src/sensor_data.jl +using Dates, DataFrames -@dataclass -class SensorReading: - sensor_id: str - timestamp: str - value: float - unit: str - metadata: Dict[str, Any] = None +struct SensorReading + sensor_id::String + timestamp::String + value::Float64 + unit::String + metadata::Dict{String, Any} +end + +function SensorReading(sensor_id::String, value::Float64, unit::String, metadata::Dict{String, Any}=Dict()) + SensorReading( + sensor_id, + ISODateTime(now(), Dates.Second) |> string, + value, + unit, + metadata + ) +end + +struct SensorBatch + readings::Vector{SensorReading} +end + +function SensorBatch() + SensorBatch(SensorReading[]) +end + +function add_reading(batch::SensorBatch, reading::SensorReading) + push!(batch.readings, reading) +end + +function to_dataframe(batch::SensorBatch)::DataFrame + data = Dict{String, Any}() + data["sensor_id"] = [r.sensor_id for r in batch.readings] + data["timestamp"] = [r.timestamp for r in batch.readings] + data["value"] = [r.value for r in batch.readings] + data["unit"] = [r.unit for r in batch.readings] - def to_dict(self) -> Dict[str, Any]: - return { - "sensor_id": self.sensor_id, - "timestamp": self.timestamp, - "value": self.value, - "unit": self.unit, - "metadata": self.metadata or {} - } - -class SensorBatch: - def __init__(self): - self.readings: List[SensorReading] = [] - - def add_reading(self, reading: SensorReading): - self.readings.append(reading) - - def to_dataframe(self): - import pandas as pd - data = [r.to_dict() for r in self.readings] - return pd.DataFrame(data) + return DataFrame(data) +end ``` -### Step 2: Sensor Sender +### Step 2: Sensor Sender (Julia) -```python -# src/sensor-sender.py -from nats_bridge import smartsend -from sensor_data import SensorReading, SensorBatch -import time -import random +```julia +# src/sensor_sender.jl +using NATSBridge, Dates, Random -class SensorSender: - def __init__(self, broker_url: str, fileserver_url: str): - self.broker_url = broker_url - self.fileserver_url = fileserver_url - - def send_reading(self, sensor_id: str, value: float, unit: str): - reading = SensorReading( - sensor_id=sensor_id, - timestamp=datetime.now().isoformat(), - value=value, - unit=unit - ) - - data = [("reading", reading.to_dict(), "dictionary")] - - # Default: is_publish=True (automatically publishes to NATS) +struct SensorSender + broker_url::String + fileserver_url::String +end + +function SensorSender(broker_url::String, fileserver_url::String) + SensorSender(broker_url, fileserver_url) +end + +function send_reading(sender::SensorSender, sensor_id::String, value::Float64, unit::String) + reading = SensorReading(sensor_id, value, unit) + + data = [("reading", reading.metadata, "dictionary")] + + # Default: is_publish=True (automatically publishes to NATS) + smartsend( + "/sensors/$sensor_id", + data, + broker_url=sender.broker_url, + fileserver_url=sender.fileserver_url + ) +end + +function prepare_message_only(sender::SensorSender, sensor_id::String, value::Float64, unit::String) + """Prepare a message without publishing (is_publish=False).""" + reading = SensorReading(sensor_id, value, unit) + + data = [("reading", reading.metadata, "dictionary")] + + # With is_publish=False, returns (env, env_json_str) without publishing + env, env_json_str = smartsend( + "/sensors/$sensor_id/prepare", + data, + broker_url=sender.broker_url, + fileserver_url=sender.fileserver_url, + is_publish=false + ) + + # Now you can publish manually using NATS request-reply pattern + # nc.request(subject, env_json_str, reply_to=reply_to_topic) + + return env, env_json_str +end + +function send_batch(sender::SensorSender, readings::Vector{SensorReading}) + batch = SensorBatch() + for reading in readings + add_reading(batch, reading) + end + + df = to_dataframe(batch) + + # Convert to Arrow IPC format + import Arrow + table = Arrow.Table(df) + + # Serialize to Arrow IPC + import IOBuffer + buf = IOBuffer() + Arrow.write(buf, table) + + arrow_data = take!(buf) + + # Send based on size + if length(arrow_data) < 1048576 # < 1MB + data = [("batch", arrow_data, "table")] smartsend( - f"/sensors/{sensor_id}", + "/sensors/batch", data, - broker_url=self.broker_url, - fileserver_url=self.fileserver_url + broker_url=sender.broker_url, + fileserver_url=sender.fileserver_url ) - - def prepare_message_only(self, sensor_id: str, value: float, unit: str): - """Prepare a message without publishing (is_publish=False).""" - reading = SensorReading( - sensor_id=sensor_id, - timestamp=datetime.now().isoformat(), - value=value, - unit=unit - ) - - data = [("reading", reading.to_dict(), "dictionary")] - - # With is_publish=False, returns (env, env_json_str) without publishing - env, env_json_str = smartsend( - f"/sensors/{sensor_id}/prepare", - data, - broker_url=self.broker_url, - fileserver_url=self.fileserver_url, - is_publish=False - ) - - # Now you can publish manually using NATS request-reply pattern - # nc.request(subject, env_json_str, reply_to=reply_to_topic) - - return env, env_json_str - - def send_batch(self, readings: List[SensorReading]): - batch = SensorBatch() - for reading in readings: - batch.add_reading(reading) - - df = batch.to_dataframe() - - # Convert to Arrow IPC format - import pyarrow as pa - table = pa.Table.from_pandas(df) - - # Serialize to Arrow IPC - import io - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - - arrow_data = buf.getvalue() - - # Send based on size - if len(arrow_data) < 1048576: # < 1MB - data = [("batch", arrow_data, "table")] - smartsend( - f"/sensors/batch", - data, - broker_url=self.broker_url, - fileserver_url=self.fileserver_url - ) - else: - # Upload to file server - from nats_bridge import plik_oneshot_upload - response = plik_oneshot_upload(self.fileserver_url, "batch.arrow", arrow_data) - print(f"Uploaded batch to {response['url']}") -``` - -### Step 3: Sensor Receiver - -```python -# src/sensor-receiver.py -from nats_bridge import smartreceive -from sensor_data import SensorReading -import pandas as pd -import pyarrow as pa -import io - -class SensorReceiver: - def __init__(self, fileserver_download_handler): - self.fileserver_download_handler = fileserver_download_handler - - def process_reading(self, msg): - env = smartreceive(msg, self.fileserver_download_handler) - - for dataname, data, data_type in env["payloads"]: - if data_type == "dictionary": - reading = SensorReading( - sensor_id=data["sensor_id"], - timestamp=data["timestamp"], - value=data["value"], - unit=data["unit"], - metadata=data.get("metadata", {}) - ) - print(f"Received: {reading}") - - elif data_type == "table": - # Deserialize Arrow IPC - table = pa.ipc.open_stream(io.BytesIO(data)).read_all() - df = table.to_pandas() - print(f"Received batch with {len(df)} readings") - print(df) -``` - ---- - -## Building a Micropython IoT Device - -Let's build an IoT device using Micropython that connects to NATS. - -### Step 1: Device Configuration - -```python -# device_config.py -import json - -class DeviceConfig: - def __init__(self, ssid, password, broker_url, device_id): - self.ssid = ssid - self.password = password - self.broker_url = broker_url - self.device_id = device_id - - def to_dict(self): - return { - "ssid": self.ssid, - "password": self.password, - "broker_url": self.broker_url, - "device_id": self.device_id - } -``` - -### Step 2: Device Bridge - -```python -# device_bridge.py -from nats_bridge import smartsend, smartreceive -import json - -class DeviceBridge: - def __init__(self, config): - self.config = config - self.broker_url = config.broker_url - - def connect(self): - # Connect to WiFi - import network - wlan = network.WLAN(network.STA_IF) - wlan.active(True) - wlan.connect(self.config.ssid, self.config.password) - - while not wlan.isconnected(): - pass - - print('Connected to WiFi') - print('Network config:', wlan.ifconfig()) - - def send_status(self, status): - data = [("status", status, "dictionary")] + else + # Upload to file server + data = [("batch", arrow_data, "table")] smartsend( - f"/devices/{self.config.device_id}/status", + "/sensors/batch", data, - broker_url=self.broker_url + broker_url=sender.broker_url, + fileserver_url=sender.fileserver_url ) - - def send_sensor_data(self, sensor_id, value, unit): - data = [ - ("sensor_id", sensor_id, "text"), - ("value", value, "dictionary") - ] - smartsend( - f"/devices/{self.config.device_id}/sensors/{sensor_id}", - data, - broker_url=self.broker_url - ) - - def receive_commands(self, callback): - # Subscribe to commands - import socket - # Simplified subscription - in production, use proper NATS client - - while True: - # Poll for messages - msg = self._poll_for_message() - if msg: - env = smartreceive(msg) - - # Process payloads - for dataname, data, data_type in env["payloads"]: - if dataname == "command": - callback(data) - - def _poll_for_message(self): - # Simplified message polling - # In production, implement proper NATS subscription - return None + end +end ``` -### Step 3: Device Application +### Step 3: Sensor Receiver (Julia) -```python -# device_app.py -from device_config import DeviceConfig -from device_bridge import DeviceBridge -import time -import random +```julia +# src/sensor_receiver.jl +using NATSBridge, Arrow, DataFrames, IOBuffer -# Load configuration -config = DeviceConfig( - ssid="MyNetwork", - password="password123", - broker_url="nats://localhost:4222", - device_id="device-001" -) +struct SensorReceiver + fileserver_download_handler::Function +end -bridge = DeviceBridge(config) -bridge.connect() +function SensorReceiver(download_handler::Function) + SensorReceiver(download_handler) +end -# Send initial status -bridge.send_status({ - "status": "online", - "version": "1.0.0", - "uptime": 0 -}) - -# Main loop -while True: - # Read sensor data - temperature = random.uniform(20, 30) - humidity = random.uniform(40, 60) +function process_reading(receiver::SensorReceiver, msg::NATS.Msg) + env = smartreceive(msg, receiver.fileserver_download_handler) - bridge.send_sensor_data("temperature", temperature, "celsius") - bridge.send_sensor_data("humidity", humidity, "percent") - - # Send status update - bridge.send_status({ - "status": "online", - "temperature": temperature, - "humidity": humidity - }) - - time.sleep(60) # Every minute -``` - ---- - -## Building a Cross-Platform Dashboard - -Let's build a dashboard that displays data from multiple platforms. - -### Step 1: Dashboard Server (Python) - -```python -# src/dashboard-server.py -from nats_bridge import smartsend, smartreceive -import pandas as pd -import pyarrow as pa -import io - -class DashboardServer: - def __init__(self, broker_url, fileserver_url): - self.broker_url = broker_url - self.fileserver_url = fileserver_url - - def broadcast_data(self, df): - # Convert to Arrow IPC - table = pa.Table.from_pandas(df) - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - - arrow_data = buf.getvalue() - - # Broadcast to all subscribers - data = [("data", arrow_data, "table")] - smartsend( - "/dashboard/data", - data, - broker_url=self.broker_url, - fileserver_url=self.fileserver_url - ) - - def receive_selection(self, callback): - def handler(msg): - env = smartreceive(msg) - - for dataname, data, data_type in env["payloads"]: - if data_type == "dictionary": - callback(data) - - # Subscribe to selections - import threading - thread = threading.Thread(target=self._listen_for_selections, args=(handler,)) - thread.daemon = True - thread.start() - - def _listen_for_selections(self, handler): - # Simplified subscription - # In production, implement proper NATS subscription - pass -``` - -### Step 2: Dashboard UI (JavaScript) - -```javascript -// src/dashboard-ui.js -class DashboardUI { - constructor() { - this.data = null; - this.setupEventListeners(); - } - - setupEventListeners() { - document.getElementById('refresh-btn').addEventListener('click', () => this.refreshData()); - document.getElementById('export-btn').addEventListener('click', () => this.exportData()); - } - - async refreshData() { - // Request fresh data - const { env, env_json_str } = await smartsend("/dashboard/request", [ - { dataname: "request", data: { type: "refresh" }, type: "dictionary" } - ], { - brokerUrl: window.config.broker_url, - fileserverUrl: window.config.fileserver_url - }); - } - - async fetchData() { - // Subscribe to data updates - const env = await smartreceive(msg, { - fileserverDownloadHandler: this.fetchFromUrl.bind(this) - }); - - // Process table data - for (const payload of env.payloads) { - if (payload.type === 'table') { - // Deserialize Arrow IPC - this.data = this.deserializeArrow(payload.data); - this.renderTable(); - } - } - } - - deserializeArrow(data) { - // Deserialize Arrow IPC to JavaScript array - // This would require the apache-arrow library - return JSON.parse(JSON.stringify(data)); // Simplified - } - - renderTable() { - const tableContainer = document.getElementById('data-table'); - tableContainer.innerHTML = ''; - - if (!this.data) return; - - // Render table headers - const headers = Object.keys(this.data[0]); - const thead = document.createElement('thead'); - const headerRow = document.createElement('tr'); - - headers.forEach(header => { - const th = document.createElement('th'); - th.textContent = header; - headerRow.appendChild(th); - }); - - thead.appendChild(headerRow); - tableContainer.appendChild(thead); - - // Render table rows - const tbody = document.createElement('tbody'); - - this.data.forEach(row => { - const tr = document.createElement('tr'); - headers.forEach(header => { - const td = document.createElement('td'); - td.textContent = row[header]; - tr.appendChild(td); - }); - tbody.appendChild(tr); - }); - - tableContainer.appendChild(tbody); - } - - exportData() { - const csv = this.toCSV(); - const blob = new Blob([csv], { type: 'text/csv' }); - const url = URL.createObjectURL(blob); - - const a = document.createElement('a'); - a.href = url; - a.download = 'dashboard_data.csv'; - a.click(); - } - - toCSV() { - if (!this.data) return ''; - - const headers = Object.keys(this.data[0]); - const rows = this.data.map(row => - headers.map(h => row[h]).join(',') - ); - - return [headers.join(','), ...rows].join('\n'); - } - - fetchFromUrl(url) { - return fetch(url) - .then(response => response.arrayBuffer()); - } -} + for (dataname, data, data_type) in env["payloads"] + if data_type == "dictionary" + # Process dictionary payload + println("Received: $dataname = $data") + elseif data_type == "table" + # Deserialize Arrow IPC + buf = IOBuffer(data) + table = Arrow.read(buf) + df = DataFrame(table) + println("Received batch with $(nrow(df)) readings") + println(df) + end + end +end ``` --- @@ -935,65 +541,84 @@ class DashboardUI { ### 1. Batch Processing -```python +```julia # Batch multiple readings into a single message -def send_batch_readings(self, readings): +function send_batch_readings(sender::SensorSender, readings::Vector{Tuple{String, Float64, String}}) batch = SensorBatch() - for reading in readings: - batch.add_reading(reading) - - df = batch.to_dataframe() + + for (sensor_id, value, unit) in readings + reading = SensorReading(sensor_id, value, unit) + add_reading(batch, reading) + end + + df = to_dataframe(batch) # Convert to Arrow IPC - table = pa.Table.from_pandas(df) - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - - arrow_data = buf.getvalue() + import Arrow + table = Arrow.Table(df) + + # Serialize to Arrow IPC + import IOBuffer + buf = IOBuffer() + Arrow.write(buf, table) + + arrow_data = take!(buf) # Send as single message smartsend( "/sensors/batch", [("batch", arrow_data, "table")], - broker_url=self.broker_url + broker_url=sender.broker_url ) +end ``` -### 2. Connection Pooling +### 2. Connection Reuse -```javascript -// Reuse NATS connections -const nats = require('nats'); - -class ConnectionPool { - constructor() { - this.connections = new Map(); - } +```julia +# Reuse NATS connections +function create_connection_pool() + connections = Dict{String, NATS.Connection}() - getConnection(natsUrl) { - if (!this.connections.has(natsUrl)) { - this.connections.set(natsUrl, nats.connect({ servers: [natsUrl] })); - } - return this.connections.get(natsUrl); - } + function get_connection(nats_url::String)::NATS.Connection + if !haskey(connections, nats_url) + connections[nats_url] = NATS.connect(nats_url) + end + return connections[nats_url] + end - closeAll() { - this.connections.forEach(conn => conn.close()); - this.connections.clear(); - } -} + function close_all() + for conn in values(connections) + NATS.drain(conn) + end + empty!(connections) + end + + return (get_connection= get_connection, close_all=close_all) +end ``` ### 3. Caching -```python +```julia # Cache file server responses -from functools import lru_cache +using Base.Threads -@lru_cache(maxsize=100) -def fetch_with_caching(url, max_retries=5, base_delay=100, max_delay=5000, correlation_id=""): - return _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) +const file_cache = Dict{String, Vector{UInt8}}() + +function fetch_with_caching(url::String, max_retries::Int, base_delay::Int, max_delay::Int, correlation_id::String)::Vector{UInt8} + if haskey(file_cache, url) + return file_cache[url] + end + + # Fetch from file server + data = _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) + + # Cache the result + file_cache[url] = data + + return data +end ``` --- @@ -1002,54 +627,61 @@ def fetch_with_caching(url, max_retries=5, base_delay=100, max_delay=5000, corre ### 1. Error Handling -```python -def safe_smartsend(subject, data, **kwargs): - try: - return smartsend(subject, data, **kwargs) - except Exception as e: - print(f"Failed to send message: {e}") - return None +```julia +function safe_smartsend(subject::String, data::Vector{Tuple}, kwargs...) + try + return smartsend(subject, data; kwargs...) + catch error + println("Failed to send message: $(error)") + return nothing + end +end ``` ### 2. Logging -```python -import logging +```julia +using Logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) +function log_send(subject::String, data::Vector{Tuple}, correlation_id::String) + @info "Sending to $subject: $(length(data)) payloads, correlation_id=$correlation_id" +end -def log_send(subject, data, correlation_id): - logger.info(f"Sending to {subject}: {len(data)} payloads, correlation_id={correlation_id}") - -def log_receive(correlation_id, num_payloads): - logger.info(f"Received message: {num_payloads} payloads, correlation_id={correlation_id}") +function log_receive(correlation_id::String, num_payloads::Int) + @info "Received message: $num_payloads payloads, correlation_id=$correlation_id" +end ``` ### 3. Rate Limiting -```python -import time -from collections import deque +```julia +using Dates, Collections -class RateLimiter: - def __init__(self, max_requests, time_window): - self.max_requests = max_requests - self.time_window = time_window - self.requests = deque() - - def allow(self): - now = time.time() - - # Remove old requests - while self.requests and self.requests[0] < now - self.time_window: - self.requests.popleft() - - if len(self.requests) >= self.max_requests: - return False - - self.requests.append(now) - return True +struct RateLimiter + max_requests::Int + time_window::Float64 + requests::Deque{Float64} +end + +function RateLimiter(max_requests::Int, time_window::Float64) + RateLimiter(max_requests, time_window, Deque{Float64}()) +end + +function allow(limiter::RateLimiter)::Bool + now = time() + + # Remove old requests + while !isempty(limiter.requests) && limiter.requests[1] < now - limiter.time_window + popfirst!(limiter.requests) + end + + if length(limiter.requests) >= limiter.max_requests + return false + end + + push!(limiter.requests, now) + return true +end ``` --- @@ -1061,8 +693,6 @@ This walkthrough covered: - Building a chat application with rich media support - Building a file transfer system with claim-check pattern - Building a streaming data pipeline for sensor data -- Building a Micropython IoT device -- Building a cross-platform dashboard For more information, check the [API documentation](../src/README.md) and [test examples](../test/). diff --git a/test/test_js_dict_receiver.js b/test/test_js_dict_receiver.js deleted file mode 100644 index 7463b9b..0000000 --- a/test/test_js_dict_receiver.js +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env node -// Test script for Dictionary transport testing -// Tests receiving 1 large and 1 small Dictionaries via direct and link transport -// Uses NATSBridge.js smartreceive with "dictionary" type - -const { smartreceive, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_dict_test"; -const NATS_URL = "nats.yiem.cc"; - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] ${message}`); -} - -// Receiver: Listen for messages and verify Dictionary handling -async function test_dict_receive() { - // Connect to NATS - const { connect } = require('nats'); - const nc = await connect({ servers: [NATS_URL] }); - - // Subscribe to the subject - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - log_trace(`Received message on ${msg.subject}`); - - // Use NATSBridge.smartreceive to handle the data - const result = await smartreceive( - msg, - { - maxRetries: 5, - baseDelay: 100, - maxDelay: 5000 - } - ); - - // Result is an envelope dictionary with payloads field - // Access payloads with result.payloads - for (const { dataname, data, type } of result.payloads) { - if (typeof data === 'object' && data !== null && !Array.isArray(data)) { - log_trace(`Received Dictionary '${dataname}' of type ${type}`); - - // Display dictionary contents - console.log(" Contents:"); - for (const [key, value] of Object.entries(data)) { - console.log(` ${key} => ${value}`); - } - - // Save to JSON file - const fs = require('fs'); - const output_path = `./received_${dataname}.json`; - const json_str = JSON.stringify(data, null, 2); - fs.writeFileSync(output_path, json_str); - log_trace(`Saved Dictionary to ${output_path}`); - } else { - log_trace(`Received unexpected data type for '${dataname}': ${typeof data}`); - } - } - } - - // Keep listening for 10 seconds - setTimeout(() => { - nc.close(); - process.exit(0); - }, 120000); -} - -// Run the test -console.log("Starting Dictionary transport test..."); -console.log("Note: This receiver will wait for messages from the sender."); -console.log("Run test_js_to_js_dict_sender.js first to send test data."); - -// Run receiver -console.log("testing smartreceive"); -test_dict_receive(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_dict_sender.js b/test/test_js_dict_sender.js deleted file mode 100644 index 6da3c21..0000000 --- a/test/test_js_dict_sender.js +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env node -// Test script for Dictionary transport testing -// Tests sending 1 large and 1 small Dictionaries via direct and link transport -// Uses NATSBridge.js smartsend with "dictionary" type - -const { smartsend, uuid4, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_dict_test"; -const NATS_URL = "nats.yiem.cc"; -const FILESERVER_URL = "http://192.168.88.104:8080"; - -// Create correlation ID for tracing -const correlation_id = uuid4(); - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); -} - -// File upload handler for plik server -async function plik_upload_handler(fileserver_url, dataname, data, correlation_id) { - // Get upload ID - const url_getUploadID = `${fileserver_url}/upload`; - const headers = { - "Content-Type": "application/json" - }; - const body = JSON.stringify({ OneShot: true }); - - let response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - if (!response.ok) { - throw new Error(`Failed to get upload ID: ${response.status} ${response.statusText}`); - } - - const responseJson = await response.json(); - const uploadid = responseJson.id; - const uploadtoken = responseJson.uploadToken; - - // Upload file - const formData = new FormData(); - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - response = await fetch(`${fileserver_url}/file/${uploadid}`, { - method: "POST", - headers: { - "X-UploadToken": uploadtoken - }, - body: formData - }); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.status} ${response.statusText}`); - } - - const fileResponseJson = await response.json(); - const fileid = fileResponseJson.id; - - const url = `${fileserver_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - return { - status: response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; -} - -// Sender: Send Dictionaries via smartsend -async function test_dict_send() { - // Create a small Dictionary (will use direct transport) - const small_dict = { - name: "Alice", - age: 30, - scores: [95, 88, 92], - metadata: { - height: 155, - weight: 55 - } - }; - - // Create a large Dictionary (will use link transport if > 1MB) - const large_dict_ids = []; - const large_dict_names = []; - const large_dict_scores = []; - const large_dict_categories = []; - - for (let i = 0; i < 50000; i++) { - large_dict_ids.push(i + 1); - large_dict_names.push(`User_${i}`); - large_dict_scores.push(Math.floor(Math.random() * 100) + 1); - large_dict_categories.push(`Category_${Math.floor(Math.random() * 10) + 1}`); - } - - const large_dict = { - ids: large_dict_ids, - names: large_dict_names, - scores: large_dict_scores, - categories: large_dict_categories, - metadata: { - source: "test_generator", - timestamp: new Date().toISOString() - } - }; - - // Test data 1: small Dictionary - const data1 = { dataname: "small_dict", data: small_dict, type: "dictionary" }; - - // Test data 2: large Dictionary - const data2 = { dataname: "large_dict", data: large_dict, type: "dictionary" }; - - // Use smartsend with dictionary type - // For small Dictionary: will use direct transport (JSON encoded) - // For large Dictionary: will use link transport (uploaded to fileserver) - const { env, env_json_str } = await smartsend( - SUBJECT, - [data1, data2], - { - natsUrl: NATS_URL, - fileserverUrl: FILESERVER_URL, - fileserverUploadHandler: plik_upload_handler, - sizeThreshold: 1_000_000, - correlationId: correlation_id, - msgPurpose: "chat", - senderName: "dict_sender", - receiverName: "", - receiverId: "", - replyTo: "", - replyToMsgId: "", - isPublish: true // Publish the message to NATS - } - ); - - log_trace(`Sent message with ${env.payloads.length} payloads`); - - // Log transport type for each payload - for (let i = 0; i < env.payloads.length; i++) { - const payload = env.payloads[i]; - log_trace(`Payload ${i + 1} ('${payload.dataname}'):`); - log_trace(` Transport: ${payload.transport}`); - log_trace(` Type: ${payload.type}`); - log_trace(` Size: ${payload.size} bytes`); - log_trace(` Encoding: ${payload.encoding}`); - - if (payload.transport === "link") { - log_trace(` URL: ${payload.data}`); - } - } -} - -// Run the test -console.log("Starting Dictionary transport test..."); -console.log(`Correlation ID: ${correlation_id}`); - -// Run sender -console.log("start smartsend for dictionaries"); -test_dict_send(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_file_receiver.js b/test/test_js_file_receiver.js deleted file mode 100644 index 4943791..0000000 --- a/test/test_js_file_receiver.js +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env node -// Test script for large payload testing using binary transport -// Tests receiving a large file (> 1MB) via smartsend with binary type - -const { smartreceive, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_test"; -const NATS_URL = "nats.yiem.cc"; - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] ${message}`); -} - -// Receiver: Listen for messages and verify large payload handling -async function test_large_binary_receive() { - // Connect to NATS - const { connect } = require('nats'); - const nc = await connect({ servers: [NATS_URL] }); - - // Subscribe to the subject - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - log_trace(`Received message on ${msg.subject}`); - - // Use NATSBridge.smartreceive to handle the data - const result = await smartreceive( - msg, - { - maxRetries: 5, - baseDelay: 100, - maxDelay: 5000 - } - ); - - // Result is an envelope dictionary with payloads field - // Access payloads with result.payloads - for (const { dataname, data, type } of result.payloads) { - if (data instanceof Uint8Array || Array.isArray(data)) { - const file_size = data.length; - log_trace(`Received ${file_size} bytes of binary data for '${dataname}' of type ${type}`); - - // Save received data to a test file - const fs = require('fs'); - const output_path = `./new_${dataname}`; - fs.writeFileSync(output_path, Buffer.from(data)); - log_trace(`Saved received data to ${output_path}`); - } else { - log_trace(`Received unexpected data type for '${dataname}': ${typeof data}`); - } - } - } - - // Keep listening for 10 seconds - setTimeout(() => { - nc.close(); - process.exit(0); - }, 120000); -} - -// Run the test -console.log("Starting large binary payload test..."); - -// Run receiver -console.log("testing smartreceive"); -test_large_binary_receive(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_file_sender.js b/test/test_js_file_sender.js deleted file mode 100644 index 383553c..0000000 --- a/test/test_js_file_sender.js +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env node -// Test script for large payload testing using binary transport -// Tests sending a large file (> 1MB) via smartsend with binary type - -const { smartsend, uuid4, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_test"; -const NATS_URL = "nats.yiem.cc"; -const FILESERVER_URL = "http://192.168.88.104:8080"; - -// Create correlation ID for tracing -const correlation_id = uuid4(); - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); -} - -// File upload handler for plik server -async function plik_upload_handler(fileserver_url, dataname, data, correlation_id) { - log_trace(correlation_id, `Uploading ${dataname} to fileserver: ${fileserver_url}`); - - // Step 1: Get upload ID and token - const url_getUploadID = `${fileserver_url}/upload`; - const headers = { - "Content-Type": "application/json" - }; - const body = JSON.stringify({ OneShot: true }); - - let response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - if (!response.ok) { - throw new Error(`Failed to get upload ID: ${response.status} ${response.statusText}`); - } - - const responseJson = await response.json(); - const uploadid = responseJson.id; - const uploadtoken = responseJson.uploadToken; - - // Step 2: Upload file data - const url_upload = `${fileserver_url}/file/${uploadid}`; - - // Create multipart form data - const formData = new FormData(); - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - response = await fetch(url_upload, { - method: "POST", - headers: { - "X-UploadToken": uploadtoken - }, - body: formData - }); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.status} ${response.statusText}`); - } - - const fileResponseJson = await response.json(); - const fileid = fileResponseJson.id; - - // Build the download URL - const url = `${fileserver_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - log_trace(correlation_id, `Uploaded to URL: ${url}`); - - return { - status: response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; -} - -// Sender: Send large binary file via smartsend -async function test_large_binary_send() { - // Read the large file as binary data - const fs = require('fs'); - - // Test data 1 - const file_path1 = './testFile_large.zip'; - const file_data1 = fs.readFileSync(file_path1); - const filename1 = 'testFile_large.zip'; - const data1 = { dataname: filename1, data: file_data1, type: "binary" }; - - // Test data 2 - const file_path2 = './testFile_small.zip'; - const file_data2 = fs.readFileSync(file_path2); - const filename2 = 'testFile_small.zip'; - const data2 = { dataname: filename2, data: file_data2, type: "binary" }; - - // Use smartsend with binary type - will automatically use link transport - // if file size exceeds the threshold (1MB by default) - const { env, env_json_str } = await smartsend( - SUBJECT, - [data1, data2], - { - natsUrl: NATS_URL, - fileserverUrl: FILESERVER_URL, - fileserverUploadHandler: plik_upload_handler, - sizeThreshold: 1_000_000, - correlationId: correlation_id, - msgPurpose: "chat", - senderName: "sender", - receiverName: "", - receiverId: "", - replyTo: "", - replyToMsgId: "", - isPublish: true // Publish the message to NATS - } - ); - - log_trace(`Sent message with transport: ${env.payloads[0].transport}`); - log_trace(`Envelope type: ${env.payloads[0].type}`); - - // Check if link transport was used - if (env.payloads[0].transport === "link") { - log_trace("Using link transport - file uploaded to HTTP server"); - log_trace(`URL: ${env.payloads[0].data}`); - } else { - log_trace("Using direct transport - payload sent via NATS"); - } -} - -// Run the test -console.log("Starting large binary payload test..."); -console.log(`Correlation ID: ${correlation_id}`); - -// Run sender first -console.log("start smartsend"); -test_large_binary_send(); - -// Run receiver -// console.log("testing smartreceive"); -// test_large_binary_receive(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_mix_payload_sender.js b/test/test_js_mix_payload_sender.js deleted file mode 100644 index 9c02b94..0000000 --- a/test/test_js_mix_payload_sender.js +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env node -// Test script for mixed-content message testing -// Tests sending a mix of text, json, table, image, audio, video, and binary data -// from JavaScript serviceA to JavaScript serviceB using NATSBridge.js smartsend -// -// This test demonstrates that any combination and any number of mixed content -// can be sent and received correctly. - -const { smartsend, uuid4, log_trace, _serialize_data } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_mix_test"; -const NATS_URL = "nats.yiem.cc"; -const FILESERVER_URL = "http://192.168.88.104:8080"; - -// Create correlation ID for tracing -const correlation_id = uuid4(); - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); -} - -// File upload handler for plik server -async function plik_upload_handler(fileserver_url, dataname, data, correlation_id) { - log_trace(correlation_id, `Uploading ${dataname} to fileserver: ${fileserver_url}`); - - // Step 1: Get upload ID and token - const url_getUploadID = `${fileserver_url}/upload`; - const headers = { - "Content-Type": "application/json" - }; - const body = JSON.stringify({ OneShot: true }); - - let response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - if (!response.ok) { - throw new Error(`Failed to get upload ID: ${response.status} ${response.statusText}`); - } - - const responseJson = await response.json(); - const uploadid = responseJson.id; - const uploadtoken = responseJson.uploadToken; - - // Step 2: Upload file data - const url_upload = `${fileserver_url}/file/${uploadid}`; - - // Create multipart form data - const formData = new FormData(); - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - response = await fetch(url_upload, { - method: "POST", - headers: { - "X-UploadToken": uploadtoken - }, - body: formData - }); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.status} ${response.statusText}`); - } - - const fileResponseJson = await response.json(); - const fileid = fileResponseJson.id; - - // Build the download URL - const url = `${fileserver_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - log_trace(correlation_id, `Uploaded to URL: ${url}`); - - return { - status: response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; -} - -// Helper: Create sample data for each type -function create_sample_data() { - // Text data (small - direct transport) - const text_data = "Hello! This is a test chat message. 🎉\nHow are you doing today? 😊"; - - // Dictionary/JSON data (medium - could be direct or link) - const dict_data = { - type: "chat", - sender: "serviceA", - receiver: "serviceB", - metadata: { - timestamp: new Date().toISOString(), - priority: "high", - tags: ["urgent", "chat", "test"] - }, - content: { - text: "This is a JSON-formatted chat message with nested structure.", - format: "markdown", - mentions: ["user1", "user2"] - } - }; - - // Table data (small - direct transport) - NOT IMPLEMENTED (requires apache-arrow) - // const table_data_small = {...}; - - // Table data (large - link transport) - NOT IMPLEMENTED (requires apache-arrow) - // const table_data_large = {...}; - - // Image data (small binary - direct transport) - // Create a simple 10x10 pixel PNG-like data - const image_width = 10; - const image_height = 10; - let image_data = new Uint8Array(128); // PNG header + pixel data - // PNG header - image_data[0] = 0x89; - image_data[1] = 0x50; - image_data[2] = 0x4E; - image_data[3] = 0x47; - image_data[4] = 0x0D; - image_data[5] = 0x0A; - image_data[6] = 0x1A; - image_data[7] = 0x0A; - // Simple RGB data (10*10*3 = 300 bytes) - for (let i = 0; i < 300; i++) { - image_data[i + 8] = 0xFF; // Red pixel - } - - // Image data (large - link transport) - const large_image_width = 500; - const large_image_height = 1000; - const large_image_data = new Uint8Array(large_image_width * large_image_height * 3 + 8); - // PNG header - large_image_data[0] = 0x89; - large_image_data[1] = 0x50; - large_image_data[2] = 0x4E; - large_image_data[3] = 0x47; - large_image_data[4] = 0x0D; - large_image_data[5] = 0x0A; - large_image_data[6] = 0x1A; - large_image_data[7] = 0x0A; - // Random RGB data - for (let i = 0; i < large_image_width * large_image_height * 3; i++) { - large_image_data[i + 8] = Math.floor(Math.random() * 255); - } - - // Audio data (small binary - direct transport) - const audio_data = new Uint8Array(100); - for (let i = 0; i < 100; i++) { - audio_data[i] = Math.floor(Math.random() * 255); - } - - // Audio data (large - link transport) - const large_audio_data = new Uint8Array(1_500_000); - for (let i = 0; i < 1_500_000; i++) { - large_audio_data[i] = Math.floor(Math.random() * 255); - } - - // Video data (small binary - direct transport) - const video_data = new Uint8Array(150); - for (let i = 0; i < 150; i++) { - video_data[i] = Math.floor(Math.random() * 255); - } - - // Video data (large - link transport) - const large_video_data = new Uint8Array(1_500_000); - for (let i = 0; i < 1_500_000; i++) { - large_video_data[i] = Math.floor(Math.random() * 255); - } - - // Binary data (small - direct transport) - const binary_data = new Uint8Array(200); - for (let i = 0; i < 200; i++) { - binary_data[i] = Math.floor(Math.random() * 255); - } - - // Binary data (large - link transport) - const large_binary_data = new Uint8Array(1_500_000); - for (let i = 0; i < 1_500_000; i++) { - large_binary_data[i] = Math.floor(Math.random() * 255); - } - - return { - text_data, - dict_data, - // table_data_small, - // table_data_large, - image_data, - large_image_data, - audio_data, - large_audio_data, - video_data, - large_video_data, - binary_data, - large_binary_data - }; -} - -// Sender: Send mixed content via smartsend -async function test_mix_send() { - // Create sample data - const { text_data, dict_data, image_data, large_image_data, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data } = create_sample_data(); - - // Create payloads list - mixed content with both small and large data - // Small data uses direct transport, large data uses link transport - const payloads = [ - // Small data (direct transport) - text, dictionary - { dataname: "chat_text", data: text_data, type: "text" }, - { dataname: "chat_json", data: dict_data, type: "dictionary" }, - // { dataname: "chat_table_small", data: table_data_small, type: "table" }, - - // Large data (link transport) - large image, large audio, large video, large binary - // { dataname: "chat_table_large", data: table_data_large, type: "table" }, - { dataname: "user_image_large", data: large_image_data, type: "image" }, - { dataname: "audio_clip_large", data: large_audio_data, type: "audio" }, - { dataname: "video_clip_large", data: large_video_data, type: "video" }, - { dataname: "binary_file_large", data: large_binary_data, type: "binary" } - ]; - - // Use smartsend with mixed content - const { env, env_json_str } = await smartsend( - SUBJECT, - payloads, - { - natsUrl: NATS_URL, - fileserverUrl: FILESERVER_URL, - fileserverUploadHandler: plik_upload_handler, - sizeThreshold: 1_000_000, - correlationId: correlation_id, - msgPurpose: "chat", - senderName: "mix_sender", - receiverName: "", - receiverId: "", - replyTo: "", - replyToMsgId: "", - isPublish: true // Publish the message to NATS - } - ); - - log_trace(`Sent message with ${env.payloads.length} payloads`); - - // Log transport type for each payload - for (let i = 0; i < env.payloads.length; i++) { - const payload = env.payloads[i]; - log_trace(`Payload ${i + 1} ('${payload.dataname}'):`); - log_trace(` Transport: ${payload.transport}`); - log_trace(` Type: ${payload.type}`); - log_trace(` Size: ${payload.size} bytes`); - log_trace(` Encoding: ${payload.encoding}`); - - if (payload.transport === "link") { - log_trace(` URL: ${payload.data}`); - } - } - - // Summary - console.log("\n--- Transport Summary ---"); - const direct_count = env.payloads.filter(p => p.transport === "direct").length; - const link_count = env.payloads.filter(p => p.transport === "link").length; - log_trace(`Direct transport: ${direct_count} payloads`); - log_trace(`Link transport: ${link_count} payloads`); -} - -// Run the test -console.log("Starting mixed-content transport test..."); -console.log(`Correlation ID: ${correlation_id}`); - -// Run sender -console.log("start smartsend for mixed content"); -test_mix_send(); - -console.log("\nTest completed."); -console.log("Note: Run test_js_to_js_mix_receiver.js to receive the messages."); \ No newline at end of file diff --git a/test/test_js_mix_payloads_receiver.js b/test/test_js_mix_payloads_receiver.js deleted file mode 100644 index 9ebf93b..0000000 --- a/test/test_js_mix_payloads_receiver.js +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env node -// Test script for mixed-content message testing -// Tests receiving a mix of text, json, table, image, audio, video, and binary data -// from JavaScript serviceA to JavaScript serviceB using NATSBridge.js smartreceive -// -// This test demonstrates that any combination and any number of mixed content -// can be sent and received correctly. - -const { smartreceive, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_mix_test"; -const NATS_URL = "nats.yiem.cc"; - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] ${message}`); -} - -// Receiver: Listen for messages and verify mixed content handling -async function test_mix_receive() { - // Connect to NATS - const { connect } = require('nats'); - const nc = await connect({ servers: [NATS_URL] }); - - // Subscribe to the subject - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - log_trace(`Received message on ${msg.subject}`); - - // Use NATSBridge.smartreceive to handle the data - const result = await smartreceive( - msg, - { - maxRetries: 5, - baseDelay: 100, - maxDelay: 5000 - } - ); - - log_trace(`Received ${result.payloads.length} payloads`); - - // Result is an envelope dictionary with payloads field - // Access payloads with result.payloads - for (const { dataname, data, type } of result.payloads) { - log_trace(`\n=== Payload: ${dataname} (type: ${type}) ===`); - - // Handle different data types - if (type === "text") { - // Text data - should be a String - if (typeof data === 'string') { - log_trace(` Type: String`); - log_trace(` Length: ${data.length} characters`); - - // Display first 200 characters - if (data.length > 200) { - log_trace(` First 200 chars: ${data.substring(0, 200)}...`); - } else { - log_trace(` Content: ${data}`); - } - - // Save to file - const fs = require('fs'); - const output_path = `./received_${dataname}.txt`; - fs.writeFileSync(output_path, data); - log_trace(` Saved to: ${output_path}`); - } else { - log_trace(` ERROR: Expected String, got ${typeof data}`); - } - - } else if (type === "dictionary") { - // Dictionary data - should be an object - if (typeof data === 'object' && data !== null && !Array.isArray(data)) { - log_trace(` Type: Object`); - log_trace(` Keys: ${Object.keys(data).join(', ')}`); - - // Display nested content - for (const [key, value] of Object.entries(data)) { - log_trace(` ${key} => ${value}`); - } - - // Save to JSON file - const fs = require('fs'); - const output_path = `./received_${dataname}.json`; - const json_str = JSON.stringify(data, null, 2); - fs.writeFileSync(output_path, json_str); - log_trace(` Saved to: ${output_path}`); - } else { - log_trace(` ERROR: Expected Object, got ${typeof data}`); - } - - } else if (type === "table") { - // Table data - should be an array of objects (requires apache-arrow) - log_trace(` Type: Array (requires apache-arrow for full deserialization)`); - if (Array.isArray(data)) { - log_trace(` Length: ${data.length} items`); - log_trace(` First item: ${JSON.stringify(data[0])}`); - } else { - log_trace(` ERROR: Expected Array, got ${typeof data}`); - } - - } else if (type === "image" || type === "audio" || type === "video" || type === "binary") { - // Binary data - should be Uint8Array - if (data instanceof Uint8Array || Array.isArray(data)) { - log_trace(` Type: Uint8Array (binary)`); - log_trace(` Size: ${data.length} bytes`); - - // Save to file - const fs = require('fs'); - const output_path = `./received_${dataname}.bin`; - fs.writeFileSync(output_path, Buffer.from(data)); - log_trace(` Saved to: ${output_path}`); - } else { - log_trace(` ERROR: Expected Uint8Array, got ${typeof data}`); - } - - } else { - log_trace(` ERROR: Unknown data type '${type}'`); - } - } - - // Summary - console.log("\n=== Verification Summary ==="); - const text_count = result.payloads.filter(x => x.type === "text").length; - const dict_count = result.payloads.filter(x => x.type === "dictionary").length; - const table_count = result.payloads.filter(x => x.type === "table").length; - const image_count = result.payloads.filter(x => x.type === "image").length; - const audio_count = result.payloads.filter(x => x.type === "audio").length; - const video_count = result.payloads.filter(x => x.type === "video").length; - const binary_count = result.payloads.filter(x => x.type === "binary").length; - - log_trace(`Text payloads: ${text_count}`); - log_trace(`Dictionary payloads: ${dict_count}`); - log_trace(`Table payloads: ${table_count}`); - log_trace(`Image payloads: ${image_count}`); - log_trace(`Audio payloads: ${audio_count}`); - log_trace(`Video payloads: ${video_count}`); - log_trace(`Binary payloads: ${binary_count}`); - - // Print transport type info for each payload if available - console.log("\n=== Payload Details ==="); - for (const { dataname, data, type } of result.payloads) { - if (["image", "audio", "video", "binary"].includes(type)) { - log_trace(`${dataname}: ${data.length} bytes (binary)`); - } else if (type === "table") { - log_trace(`${dataname}: ${data.length} items (Array)`); - } else if (type === "dictionary") { - log_trace(`${dataname}: ${JSON.stringify(data).length} bytes (Object)`); - } else if (type === "text") { - log_trace(`${dataname}: ${data.length} characters (String)`); - } - } - } - - // Keep listening for 2 minutes - setTimeout(() => { - nc.close(); - process.exit(0); - }, 120000); -} - -// Run the test -console.log("Starting mixed-content transport test..."); -console.log("Note: This receiver will wait for messages from the sender."); -console.log("Run test_js_to_js_mix_sender.js first to send test data."); - -// Run receiver -console.log("\ntesting smartreceive for mixed content"); -test_mix_receive(); - -console.log("\nTest completed."); \ No newline at end of file diff --git a/test/test_js_table_receiver.js b/test/test_js_table_receiver.js deleted file mode 100644 index f5672f0..0000000 --- a/test/test_js_table_receiver.js +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env node -// Test script for Table transport testing -// Tests receiving 1 large and 1 small Tables via direct and link transport -// Uses NATSBridge.js smartreceive with "table" type -// -// Note: This test requires the apache-arrow library to deserialize table data. -// The JavaScript implementation uses apache-arrow for Arrow IPC deserialization. - -const { smartreceive, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_table_test"; -const NATS_URL = "nats.yiem.cc"; - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] ${message}`); -} - -// Receiver: Listen for messages and verify Table handling -async function test_table_receive() { - // Connect to NATS - const { connect } = require('nats'); - const nc = await connect({ servers: [NATS_URL] }); - - // Subscribe to the subject - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - log_trace(`Received message on ${msg.subject}`); - - // Use NATSBridge.smartreceive to handle the data - const result = await smartreceive( - msg, - { - maxRetries: 5, - baseDelay: 100, - maxDelay: 5000 - } - ); - - // Result is an envelope dictionary with payloads field - // Access payloads with result.payloads - for (const { dataname, data, type } of result.payloads) { - if (Array.isArray(data)) { - log_trace(`Received Table '${dataname}' of type ${type}`); - - // Display table contents - console.log(` Dimensions: ${data.length} rows x ${data.length > 0 ? Object.keys(data[0]).length : 0} columns`); - console.log(` Columns: ${data.length > 0 ? Object.keys(data[0]).join(', ') : ''}`); - - // Display first few rows - console.log(` First 5 rows:`); - for (let i = 0; i < Math.min(5, data.length); i++) { - console.log(` Row ${i}: ${JSON.stringify(data[i])}`); - } - - // Save to JSON file - const fs = require('fs'); - const output_path = `./received_${dataname}.json`; - const json_str = JSON.stringify(data, null, 2); - fs.writeFileSync(output_path, json_str); - log_trace(`Saved Table to ${output_path}`); - } else { - log_trace(`Received unexpected data type for '${dataname}': ${typeof data}`); - } - } - } - - // Keep listening for 10 seconds - setTimeout(() => { - nc.close(); - process.exit(0); - }, 120000); -} - -// Run the test -console.log("Starting Table transport test..."); -console.log("Note: This receiver will wait for messages from the sender."); -console.log("Run test_js_to_js_table_sender.js first to send test data."); - -// Run receiver -console.log("testing smartreceive"); -test_table_receive(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_table_sender.js b/test/test_js_table_sender.js deleted file mode 100644 index 0ebb1b3..0000000 --- a/test/test_js_table_sender.js +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env node -// Test script for Table transport testing -// Tests sending 1 large and 1 small Tables via direct and link transport -// Uses NATSBridge.js smartsend with "table" type -// -// Note: This test requires the apache-arrow library to serialize/deserialize table data. -// The JavaScript implementation uses apache-arrow for Arrow IPC serialization. - -const { smartsend, uuid4, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_table_test"; -const NATS_URL = "nats.yiem.cc"; -const FILESERVER_URL = "http://192.168.88.104:8080"; - -// Create correlation ID for tracing -const correlation_id = uuid4(); - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); -} - -// File upload handler for plik server -async function plik_upload_handler(fileserver_url, dataname, data, correlation_id) { - log_trace(correlation_id, `Uploading ${dataname} to fileserver: ${fileserver_url}`); - - // Step 1: Get upload ID and token - const url_getUploadID = `${fileserver_url}/upload`; - const headers = { - "Content-Type": "application/json" - }; - const body = JSON.stringify({ OneShot: true }); - - let response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - if (!response.ok) { - throw new Error(`Failed to get upload ID: ${response.status} ${response.statusText}`); - } - - const responseJson = await response.json(); - const uploadid = responseJson.id; - const uploadtoken = responseJson.uploadToken; - - // Step 2: Upload file data - const url_upload = `${fileserver_url}/file/${uploadid}`; - - // Create multipart form data - const formData = new FormData(); - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - response = await fetch(url_upload, { - method: "POST", - headers: { - "X-UploadToken": uploadtoken - }, - body: formData - }); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.status} ${response.statusText}`); - } - - const fileResponseJson = await response.json(); - const fileid = fileResponseJson.id; - - // Build the download URL - const url = `${fileserver_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - log_trace(correlation_id, `Uploaded to URL: ${url}`); - - return { - status: response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; -} - -// Sender: Send Tables via smartsend -async function test_table_send() { - // Note: This test requires apache-arrow library to create Arrow IPC data. - // For now, we'll use a simple array of objects as table data. - // In production, you would use the apache-arrow library to create Arrow IPC data. - - // Create a small Table (will use direct transport) - const small_table = [ - { id: 1, name: "Alice", score: 95 }, - { id: 2, name: "Bob", score: 88 }, - { id: 3, name: "Charlie", score: 92 } - ]; - - // Create a large Table (will use link transport if > 1MB) - // Generate a larger dataset (~2MB to ensure link transport) - const large_table = []; - for (let i = 0; i < 50000; i++) { - large_table.push({ - id: i, - message: `msg_${i}`, - sender: `sender_${i}`, - timestamp: new Date().toISOString(), - priority: Math.floor(Math.random() * 3) + 1 - }); - } - - // Test data 1: small Table - const data1 = { dataname: "small_table", data: small_table, type: "table" }; - - // Test data 2: large Table - const data2 = { dataname: "large_table", data: large_table, type: "table" }; - - // Use smartsend with table type - // For small Table: will use direct transport (Arrow IPC encoded) - // For large Table: will use link transport (uploaded to fileserver) - const { env, env_json_str } = await smartsend( - SUBJECT, - [data1, data2], - { - natsUrl: NATS_URL, - fileserverUrl: FILESERVER_URL, - fileserverUploadHandler: plik_upload_handler, - sizeThreshold: 1_000_000, - correlationId: correlation_id, - msgPurpose: "chat", - senderName: "table_sender", - receiverName: "", - receiverId: "", - replyTo: "", - replyToMsgId: "", - isPublish: true // Publish the message to NATS - } - ); - - log_trace(`Sent message with ${env.payloads.length} payloads`); - - // Log transport type for each payload - for (let i = 0; i < env.payloads.length; i++) { - const payload = env.payloads[i]; - log_trace(`Payload ${i + 1} ('${payload.dataname}'):`); - log_trace(` Transport: ${payload.transport}`); - log_trace(` Type: ${payload.type}`); - log_trace(` Size: ${payload.size} bytes`); - log_trace(` Encoding: ${payload.encoding}`); - - if (payload.transport === "link") { - log_trace(` URL: ${payload.data}`); - } - } -} - -// Run the test -console.log("Starting Table transport test..."); -console.log(`Correlation ID: ${correlation_id}`); - -// Run sender -console.log("start smartsend for tables"); -test_table_send(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_text_receiver.js b/test/test_js_text_receiver.js deleted file mode 100644 index ef46822..0000000 --- a/test/test_js_text_receiver.js +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env node -// Test script for text transport testing -// Tests receiving 1 large and 1 small text from JavaScript serviceA to JavaScript serviceB -// Uses NATSBridge.js smartreceive with "text" type - -const { smartreceive, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_text_test"; -const NATS_URL = "nats.yiem.cc"; - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] ${message}`); -} - -// Receiver: Listen for messages and verify text handling -async function test_text_receive() { - // Connect to NATS - const { connect } = require('nats'); - const nc = await connect({ servers: [NATS_URL] }); - - // Subscribe to the subject - const sub = nc.subscribe(SUBJECT); - - for await (const msg of sub) { - log_trace(`Received message on ${msg.subject}`); - - // Use NATSBridge.smartreceive to handle the data - const result = await smartreceive( - msg, - { - maxRetries: 5, - baseDelay: 100, - maxDelay: 5000 - } - ); - - // Result is an envelope dictionary with payloads field - // Access payloads with result.payloads - for (const { dataname, data, type } of result.payloads) { - if (typeof data === 'string') { - log_trace(`Received text '${dataname}' of type ${type}`); - log_trace(` Length: ${data.length} characters`); - - // Display first 100 characters - if (data.length > 100) { - log_trace(` First 100 characters: ${data.substring(0, 100)}...`); - } else { - log_trace(` Content: ${data}`); - } - - // Save to file - const fs = require('fs'); - const output_path = `./received_${dataname}.txt`; - fs.writeFileSync(output_path, data); - log_trace(`Saved text to ${output_path}`); - } else { - log_trace(`Received unexpected data type for '${dataname}': ${typeof data}`); - } - } - } - - // Keep listening for 10 seconds - setTimeout(() => { - nc.close(); - process.exit(0); - }, 120000); -} - -// Run the test -console.log("Starting text transport test..."); -console.log("Note: This receiver will wait for messages from the sender."); -console.log("Run test_js_to_js_text_sender.js first to send test data."); - -// Run receiver -console.log("testing smartreceive for text"); -test_text_receive(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_js_text_sender.js b/test/test_js_text_sender.js deleted file mode 100644 index 83e05cc..0000000 --- a/test/test_js_text_sender.js +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env node -// Test script for text transport testing -// Tests sending 1 large and 1 small text from JavaScript serviceA to JavaScript serviceB -// Uses NATSBridge.js smartsend with "text" type - -const { smartsend, uuid4, log_trace } = require('./src/NATSBridge'); - -// Configuration -const SUBJECT = "/NATSBridge_text_test"; -const NATS_URL = "nats.yiem.cc"; -const FILESERVER_URL = "http://192.168.88.104:8080"; - -// Create correlation ID for tracing -const correlation_id = uuid4(); - -// Helper: Log with correlation ID -function log_trace(message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); -} - -// File upload handler for plik server -async function plik_upload_handler(fileserver_url, dataname, data, correlation_id) { - // Get upload ID - const url_getUploadID = `${fileserver_url}/upload`; - const headers = { - "Content-Type": "application/json" - }; - const body = JSON.stringify({ OneShot: true }); - - let response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - if (!response.ok) { - throw new Error(`Failed to get upload ID: ${response.status} ${response.statusText}`); - } - - const responseJson = await response.json(); - const uploadid = responseJson.id; - const uploadtoken = responseJson.uploadToken; - - // Upload file - const formData = new FormData(); - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - response = await fetch(`${fileserver_url}/file/${uploadid}`, { - method: "POST", - headers: { - "X-UploadToken": uploadtoken - }, - body: formData - }); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.status} ${response.statusText}`); - } - - const fileResponseJson = await response.json(); - const fileid = fileResponseJson.id; - - const url = `${fileserver_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - return { - status: response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; -} - -// Sender: Send text via smartsend -async function test_text_send() { - // Create a small text (will use direct transport) - const small_text = "Hello, this is a small text message. Testing direct transport via NATS."; - - // Create a large text (will use link transport if > 1MB) - // Generate a larger text (~2MB to ensure link transport) - const large_text_lines = []; - for (let i = 0; i < 50000; i++) { - large_text_lines.push(`Line ${i}: This is a sample text line with some content to pad the size. `); - } - const large_text = large_text_lines.join(""); - - // Test data 1: small text - const data1 = { dataname: "small_text", data: small_text, type: "text" }; - - // Test data 2: large text - const data2 = { dataname: "large_text", data: large_text, type: "text" }; - - // Use smartsend with text type - // For small text: will use direct transport (Base64 encoded UTF-8) - // For large text: will use link transport (uploaded to fileserver) - const { env, env_json_str } = await smartsend( - SUBJECT, - [data1, data2], - { - natsUrl: NATS_URL, - fileserverUrl: FILESERVER_URL, - fileserverUploadHandler: plik_upload_handler, - sizeThreshold: 1_000_000, - correlationId: correlation_id, - msgPurpose: "chat", - senderName: "text_sender", - receiverName: "", - receiverId: "", - replyTo: "", - replyToMsgId: "", - isPublish: true // Publish the message to NATS - } - ); - - log_trace(`Sent message with ${env.payloads.length} payloads`); - - // Log transport type for each payload - for (let i = 0; i < env.payloads.length; i++) { - const payload = env.payloads[i]; - log_trace(`Payload ${i + 1} ('${payload.dataname}'):`); - log_trace(` Transport: ${payload.transport}`); - log_trace(` Type: ${payload.type}`); - log_trace(` Size: ${payload.size} bytes`); - log_trace(` Encoding: ${payload.encoding}`); - - if (payload.transport === "link") { - log_trace(` URL: ${payload.data}`); - } - } -} - -// Run the test -console.log("Starting text transport test..."); -console.log(`Correlation ID: ${correlation_id}`); - -// Run sender -console.log("start smartsend for text"); -test_text_send(); - -console.log("Test completed."); \ No newline at end of file diff --git a/test/test_micropython_basic.py b/test/test_micropython_basic.py deleted file mode 100644 index 31884cb..0000000 --- a/test/test_micropython_basic.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -""" -Basic functionality test for nats_bridge.py -Tests the core classes and functions without NATS connection -""" - -import sys -import os - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import ( - MessagePayload, - MessageEnvelope, - smartsend, - smartreceive, - log_trace, - generate_uuid, - get_timestamp, - _serialize_data, - _deserialize_data -) -import json - - -def test_message_payload(): - """Test MessagePayload class""" - print("\n=== Testing MessagePayload ===") - - # Test direct transport with text - payload1 = MessagePayload( - data="Hello World", - msg_type="text", - id="test-id-1", - dataname="message", - transport="direct", - encoding="base64", - size=11 - ) - - assert payload1.id == "test-id-1" - assert payload1.dataname == "message" - assert payload1.type == "text" - assert payload1.transport == "direct" - assert payload1.encoding == "base64" - assert payload1.size == 11 - print(" [PASS] MessagePayload with text data") - - # Test link transport with URL - payload2 = MessagePayload( - data="http://example.com/file.txt", - msg_type="binary", - id="test-id-2", - dataname="file", - transport="link", - encoding="none", - size=1000 - ) - - assert payload2.transport == "link" - assert payload2.data == "http://example.com/file.txt" - print(" [PASS] MessagePayload with link transport") - - # Test to_dict method - payload_dict = payload1.to_dict() - assert "id" in payload_dict - assert "dataname" in payload_dict - assert "type" in payload_dict - assert "transport" in payload_dict - assert "data" in payload_dict - print(" [PASS] MessagePayload.to_dict() method") - - -def test_message_envelope(): - """Test MessageEnvelope class""" - print("\n=== Testing MessageEnvelope ===") - - # Create payloads - payload1 = MessagePayload("Hello", "text", id="p1", dataname="msg1") - payload2 = MessagePayload("http://example.com/file", "binary", id="p2", dataname="file", transport="link") - - # Create envelope - env = MessageEnvelope( - send_to="/test/subject", - payloads=[payload1, payload2], - correlation_id="test-correlation-id", - msg_id="test-msg-id", - msg_purpose="chat", - sender_name="test_sender", - receiver_name="test_receiver", - reply_to="/test/reply" - ) - - assert env.send_to == "/test/subject" - assert env.correlation_id == "test-correlation-id" - assert env.msg_id == "test-msg-id" - assert env.msg_purpose == "chat" - assert len(env.payloads) == 2 - print(" [PASS] MessageEnvelope creation") - - # Test to_json method - json_str = env.to_json() - json_data = json.loads(json_str) - assert json_data["sendTo"] == "/test/subject" - assert json_data["correlationId"] == "test-correlation-id" - assert json_data["msgPurpose"] == "chat" - assert len(json_data["payloads"]) == 2 - print(" [PASS] MessageEnvelope.to_json() method") - - -def test_serialize_data(): - """Test _serialize_data function""" - print("\n=== Testing _serialize_data ===") - - # Test text serialization - text_bytes = _serialize_data("Hello", "text") - assert isinstance(text_bytes, bytes) - assert text_bytes == b"Hello" - print(" [PASS] Text serialization") - - # Test dictionary serialization - dict_data = {"key": "value", "number": 42} - dict_bytes = _serialize_data(dict_data, "dictionary") - assert isinstance(dict_bytes, bytes) - parsed = json.loads(dict_bytes.decode('utf-8')) - assert parsed["key"] == "value" - print(" [PASS] Dictionary serialization") - - # Test binary serialization - binary_data = b"\x00\x01\x02" - binary_bytes = _serialize_data(binary_data, "binary") - assert binary_bytes == b"\x00\x01\x02" - print(" [PASS] Binary serialization") - - # Test image serialization - image_data = bytes([1, 2, 3, 4, 5]) - image_bytes = _serialize_data(image_data, "image") - assert image_bytes == image_data - print(" [PASS] Image serialization") - - -def test_deserialize_data(): - """Test _deserialize_data function""" - print("\n=== Testing _deserialize_data ===") - - # Test text deserialization - text_bytes = b"Hello" - text_data = _deserialize_data(text_bytes, "text", "test-correlation-id") - assert text_data == "Hello" - print(" [PASS] Text deserialization") - - # Test dictionary deserialization - dict_bytes = b'{"key": "value"}' - dict_data = _deserialize_data(dict_bytes, "dictionary", "test-correlation-id") - assert dict_data == {"key": "value"} - print(" [PASS] Dictionary deserialization") - - # Test binary deserialization - binary_data = b"\x00\x01\x02" - binary_result = _deserialize_data(binary_data, "binary", "test-correlation-id") - assert binary_result == b"\x00\x01\x02" - print(" [PASS] Binary deserialization") - - -def test_utilities(): - """Test utility functions""" - print("\n=== Testing Utility Functions ===") - - # Test generate_uuid - uuid1 = generate_uuid() - uuid2 = generate_uuid() - assert uuid1 != uuid2 - print(f" [PASS] generate_uuid() - generated: {uuid1}") - - # Test get_timestamp - timestamp = get_timestamp() - assert "T" in timestamp - print(f" [PASS] get_timestamp() - generated: {timestamp}") - - -def main(): - """Run all tests""" - print("=" * 60) - print("NATSBridge Python/Micropython - Basic Functionality Tests") - print("=" * 60) - - try: - test_message_payload() - test_message_envelope() - test_serialize_data() - test_deserialize_data() - test_utilities() - - print("\n" + "=" * 60) - print("ALL TESTS PASSED!") - print("=" * 60) - - except Exception as e: - print(f"\n[FAIL] Test failed with error: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test/test_micropython_dict_receiver.py b/test/test_micropython_dict_receiver.py deleted file mode 100644 index ae6df9f..0000000 --- a/test/test_micropython_dict_receiver.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for dictionary transport testing - Receiver -Tests receiving dictionary messages via NATS using nats_bridge.py smartreceive -""" - -import sys -import os -import json - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartreceive, log_trace -import nats -import asyncio - -# Configuration -SUBJECT = "/NATSBridge_dict_test" -NATS_URL = "nats://nats.yiem.cc:4222" - - -async def main(): - log_trace("", f"Starting dictionary transport receiver test...") - log_trace("", f"Note: This receiver will wait for messages from the sender.") - log_trace("", f"Run test_micropython_dict_sender.py first to send test data.") - - # Connect to NATS - nc = await nats.connect(NATS_URL) - log_trace("", f"Connected to NATS at {NATS_URL}") - - # Subscribe to the subject - async def message_handler(msg): - log_trace("", f"Received message on {msg.subject}") - - # Use smartreceive to handle the data - result = smartreceive(msg.data) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for dataname, data, data_type in result["payloads"]: - if isinstance(data, dict): - log_trace(result.get("correlationId", ""), f"Received dictionary '{dataname}' of type {data_type}") - log_trace(result.get("correlationId", ""), f" Keys: {list(data.keys())}") - - # Display first few items for small dicts - if isinstance(data, dict) and len(data) <= 10: - log_trace(result.get("correlationId", ""), f" Content: {json.dumps(data, indent=2)}") - else: - # For large dicts, show summary - log_trace(result.get("correlationId", ""), f" Summary: {json.dumps(data, default=str)[:200]}...") - - # Save to file - output_path = f"./received_{dataname}.json" - with open(output_path, 'w') as f: - json.dump(data, f, indent=2) - log_trace(result.get("correlationId", ""), f"Saved dictionary to {output_path}") - else: - log_trace(result.get("correlationId", ""), f"Received unexpected data type for '{dataname}': {type(data)}") - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - log_trace("", f"Subscribed to {SUBJECT} with subscription ID: {sid}") - - # Keep listening for 120 seconds - await asyncio.sleep(120) - await nc.close() - log_trace("", "Test completed.") - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/test/test_micropython_dict_sender.py b/test/test_micropython_dict_sender.py deleted file mode 100644 index bdc5878..0000000 --- a/test/test_micropython_dict_sender.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for dictionary transport testing - Micropython -Tests sending dictionary messages via NATS using nats_bridge.py smartsend -""" - -import sys -import os - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartsend, log_trace -import uuid - -# Configuration -SUBJECT = "/NATSBridge_dict_test" -NATS_URL = "nats://nats.yiem.cc:4222" -FILESERVER_URL = "http://192.168.88.104:8080" -SIZE_THRESHOLD = 1_000_000 # 1MB - -# Create correlation ID for tracing -correlation_id = str(uuid.uuid4()) - - -def main(): - # Create a small dictionary (will use direct transport) - small_dict = { - "name": "test", - "value": 42, - "enabled": True, - "metadata": { - "version": "1.0.0", - "timestamp": "2026-02-22T12:00:00Z" - } - } - - # Create a large dictionary (will use link transport if > 1MB) - # Generate a larger dictionary (~2MB to ensure link transport) - large_dict = { - "id": str(uuid.uuid4()), - "items": [ - { - "index": i, - "name": f"item_{i}", - "value": i * 1.5, - "data": "x" * 10000 # Large string per item - } - for i in range(200) - ], - "metadata": { - "count": 200, - "created": "2026-02-22T12:00:00Z" - } - } - - # Test data 1: small dictionary - data1 = ("small_dict", small_dict, "dictionary") - - # Test data 2: large dictionary - data2 = ("large_dict", large_dict, "dictionary") - - log_trace(correlation_id, f"Starting smartsend for subject: {SUBJECT}") - log_trace(correlation_id, f"Correlation ID: {correlation_id}") - - # Use smartsend with dictionary type - env, env_json_str = smartsend( - SUBJECT, - [data1, data2], # List of (dataname, data, type) tuples - nats_url=NATS_URL, - fileserver_url=FILESERVER_URL, - size_threshold=SIZE_THRESHOLD, - correlation_id=correlation_id, - msg_purpose="chat", - sender_name="dict_sender", - receiver_name="", - receiver_id="", - reply_to="", - reply_to_msg_id="", - is_publish=True # Publish the message to NATS - ) - - log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") - - # Log transport type for each payload - for i, payload in enumerate(env.payloads): - log_trace(correlation_id, f"Payload {i+1} ('{payload.dataname}'):") - log_trace(correlation_id, f" Transport: {payload.transport}") - log_trace(correlation_id, f" Type: {payload.type}") - log_trace(correlation_id, f" Size: {payload.size} bytes") - log_trace(correlation_id, f" Encoding: {payload.encoding}") - - if payload.transport == "link": - log_trace(correlation_id, f" URL: {payload.data}") - - print(f"Test completed. Correlation ID: {correlation_id}") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test/test_micropython_file_receiver.py b/test/test_micropython_file_receiver.py deleted file mode 100644 index 98a6ebc..0000000 --- a/test/test_micropython_file_receiver.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for file transport testing - Receiver -Tests receiving binary files via NATS using nats_bridge.py smartreceive -""" - -import sys -import os - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartreceive, log_trace -import nats -import asyncio - -# Configuration -SUBJECT = "/NATSBridge_file_test" -NATS_URL = "nats://nats.yiem.cc:4222" - - -async def main(): - log_trace("", f"Starting file transport receiver test...") - log_trace("", f"Note: This receiver will wait for messages from the sender.") - log_trace("", f"Run test_micropython_file_sender.py first to send test data.") - - # Connect to NATS - nc = await nats.connect(NATS_URL) - log_trace("", f"Connected to NATS at {NATS_URL}") - - # Subscribe to the subject - async def message_handler(msg): - log_trace("", f"Received message on {msg.subject}") - - # Use smartreceive to handle the data - result = smartreceive(msg.data) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for dataname, data, data_type in result["payloads"]: - if isinstance(data, bytes): - log_trace(result.get("correlationId", ""), f"Received binary '{dataname}' of type {data_type}") - log_trace(result.get("correlationId", ""), f" Size: {len(data)} bytes") - - # Display first 100 bytes as hex - log_trace(result.get("correlationId", ""), f" First 100 bytes (hex): {data[:100].hex()}") - - # Save to file - output_path = f"./received_{dataname}.bin" - with open(output_path, 'wb') as f: - f.write(data) - log_trace(result.get("correlationId", ""), f"Saved binary to {output_path}") - else: - log_trace(result.get("correlationId", ""), f"Received unexpected data type for '{dataname}': {type(data)}") - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - log_trace("", f"Subscribed to {SUBJECT} with subscription ID: {sid}") - - # Keep listening for 120 seconds - await asyncio.sleep(120) - await nc.close() - log_trace("", "Test completed.") - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/test/test_micropython_file_sender.py b/test/test_micropython_file_sender.py deleted file mode 100644 index 55b53bb..0000000 --- a/test/test_micropython_file_sender.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for file transport testing - Micropython -Tests sending binary files via NATS using nats_bridge.py smartsend -""" - -import sys -import os - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartsend, log_trace -import uuid - -# Configuration -SUBJECT = "/NATSBridge_file_test" -NATS_URL = "nats://nats.yiem.cc:4222" -FILESERVER_URL = "http://192.168.88.104:8080" -SIZE_THRESHOLD = 1_000_000 # 1MB - -# Create correlation ID for tracing -correlation_id = str(uuid.uuid4()) - - -def main(): - # Create small binary data (will use direct transport) - small_binary = b"This is small binary data for testing direct transport." - small_binary += b"\x00" * 100 # Add some null bytes - - # Create large binary data (will use link transport if > 1MB) - # Generate a larger binary (~2MB to ensure link transport) - large_binary = bytes([ - (i * 7) % 256 for i in range(2_000_000) - ]) - - # Test data 1: small binary (direct transport) - data1 = ("small_binary", small_binary, "binary") - - # Test data 2: large binary (link transport) - data2 = ("large_binary", large_binary, "binary") - - log_trace(correlation_id, f"Starting smartsend for subject: {SUBJECT}") - log_trace(correlation_id, f"Correlation ID: {correlation_id}") - - # Use smartsend with binary type - env, env_json_str = smartsend( - SUBJECT, - [data1, data2], # List of (dataname, data, type) tuples - nats_url=NATS_URL, - fileserver_url=FILESERVER_URL, - size_threshold=SIZE_THRESHOLD, - correlation_id=correlation_id, - msg_purpose="chat", - sender_name="file_sender", - receiver_name="", - receiver_id="", - reply_to="", - reply_to_msg_id="", - is_publish=True # Publish the message to NATS - ) - - log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") - - # Log transport type for each payload - for i, payload in enumerate(env.payloads): - log_trace(correlation_id, f"Payload {i+1} ('{payload.dataname}'):") - log_trace(correlation_id, f" Transport: {payload.transport}") - log_trace(correlation_id, f" Type: {payload.type}") - log_trace(correlation_id, f" Size: {payload.size} bytes") - log_trace(correlation_id, f" Encoding: {payload.encoding}") - - if payload.transport == "link": - log_trace(correlation_id, f" URL: {payload.data}") - - print(f"Test completed. Correlation ID: {correlation_id}") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test/test_micropython_mixed_receiver.py b/test/test_micropython_mixed_receiver.py deleted file mode 100644 index d28722d..0000000 --- a/test/test_micropython_mixed_receiver.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for mixed payload testing - Receiver -Tests receiving mixed payload types via NATS using nats_bridge.py smartreceive -""" - -import sys -import os -import json - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartreceive, log_trace -import nats -import asyncio - -# Configuration -SUBJECT = "/NATSBridge_mixed_test" -NATS_URL = "nats://nats.yiem.cc:4222" - - -async def main(): - log_trace("", f"Starting mixed payload receiver test...") - log_trace("", f"Note: This receiver will wait for messages from the sender.") - log_trace("", f"Run test_micropython_mixed_sender.py first to send test data.") - - # Connect to NATS - nc = await nats.connect(NATS_URL) - log_trace("", f"Connected to NATS at {NATS_URL}") - - # Subscribe to the subject - async def message_handler(msg): - log_trace("", f"Received message on {msg.subject}") - - # Use smartreceive to handle the data - result = smartreceive(msg.data) - - log_trace(result.get("correlationId", ""), f"Received envelope with {len(result['payloads'])} payloads") - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for dataname, data, data_type in result["payloads"]: - log_trace(result.get("correlationId", ""), f"\n--- Payload: {dataname} (type: {data_type}) ---") - - if isinstance(data, str): - log_trace(result.get("correlationId", ""), f" Type: text/string") - log_trace(result.get("correlationId", ""), f" Length: {len(data)} characters") - if len(data) <= 100: - log_trace(result.get("correlationId", ""), f" Content: {data}") - else: - log_trace(result.get("correlationId", ""), f" First 100 chars: {data[:100]}...") - # Save to file - output_path = f"./received_{dataname}.txt" - with open(output_path, 'w') as f: - f.write(data) - log_trace(result.get("correlationId", ""), f" Saved to: {output_path}") - - elif isinstance(data, dict): - log_trace(result.get("correlationId", ""), f" Type: dictionary") - log_trace(result.get("correlationId", ""), f" Keys: {list(data.keys())}") - log_trace(result.get("correlationId", ""), f" Content: {json.dumps(data, indent=2)}") - # Save to file - output_path = f"./received_{dataname}.json" - with open(output_path, 'w') as f: - json.dump(data, f, indent=2) - log_trace(result.get("correlationId", ""), f" Saved to: {output_path}") - - elif isinstance(data, bytes): - log_trace(result.get("correlationId", ""), f" Type: binary") - log_trace(result.get("correlationId", ""), f" Size: {len(data)} bytes") - log_trace(result.get("correlationId", ""), f" First 100 bytes (hex): {data[:100].hex()}") - # Save to file - output_path = f"./received_{dataname}.bin" - with open(output_path, 'wb') as f: - f.write(data) - log_trace(result.get("correlationId", ""), f" Saved to: {output_path}") - else: - log_trace(result.get("correlationId", ""), f" Received unexpected data type: {type(data)}") - - # Log envelope metadata - log_trace(result.get("correlationId", ""), f"\n--- Envelope Metadata ---") - log_trace(result.get("correlationId", ""), f" Correlation ID: {result.get('correlationId', 'N/A')}") - log_trace(result.get("correlationId", ""), f" Message ID: {result.get('msgId', 'N/A')}") - log_trace(result.get("correlationId", ""), f" Sender: {result.get('senderName', 'N/A')}") - log_trace(result.get("correlationId", ""), f" Purpose: {result.get('msgPurpose', 'N/A')}") - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - log_trace("", f"Subscribed to {SUBJECT} with subscription ID: {sid}") - - # Keep listening for 120 seconds - await asyncio.sleep(120) - await nc.close() - log_trace("", "Test completed.") - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/test/test_micropython_mixed_sender.py b/test/test_micropython_mixed_sender.py deleted file mode 100644 index 66a57a3..0000000 --- a/test/test_micropython_mixed_sender.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for mixed payload testing - Micropython -Tests sending mixed payload types via NATS using nats_bridge.py smartsend -""" - -import sys -import os - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartsend, log_trace -import uuid - -# Configuration -SUBJECT = "/NATSBridge_mixed_test" -NATS_URL = "nats://nats.yiem.cc:4222" -FILESERVER_URL = "http://192.168.88.104:8080" -SIZE_THRESHOLD = 1_000_000 # 1MB - -# Create correlation ID for tracing -correlation_id = str(uuid.uuid4()) - - -def main(): - # Create payloads for mixed content test - - # 1. Small text (direct transport) - text_data = "Hello, this is a text message for testing mixed payloads!" - - # 2. Small dictionary (direct transport) - dict_data = { - "status": "ok", - "code": 200, - "message": "Test successful", - "items": [1, 2, 3] - } - - # 3. Small binary (direct transport) - binary_data = b"\x00\x01\x02\x03\x04\x05" + b"\xff" * 100 - - # 4. Large text (link transport - will use fileserver) - large_text = "\n".join([ - f"Line {i}: This is a large text payload for link transport testing. " * 50 - for i in range(100) - ]) - - # Test data list - mixed payload types - data = [ - ("message_text", text_data, "text"), - ("config_dict", dict_data, "dictionary"), - ("small_binary", binary_data, "binary"), - ("large_text", large_text, "text"), - ] - - log_trace(correlation_id, f"Starting smartsend for subject: {SUBJECT}") - log_trace(correlation_id, f"Correlation ID: {correlation_id}") - - # Use smartsend with mixed types - env, env_json_str = smartsend( - SUBJECT, - data, # List of (dataname, data, type) tuples - nats_url=NATS_URL, - fileserver_url=FILESERVER_URL, - size_threshold=SIZE_THRESHOLD, - correlation_id=correlation_id, - msg_purpose="chat", - sender_name="mixed_sender", - receiver_name="", - receiver_id="", - reply_to="", - reply_to_msg_id="", - is_publish=True # Publish the message to NATS - ) - - log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") - - # Log transport type for each payload - for i, payload in enumerate(env.payloads): - log_trace(correlation_id, f"Payload {i+1} ('{payload.dataname}'):") - log_trace(correlation_id, f" Transport: {payload.transport}") - log_trace(correlation_id, f" Type: {payload.type}") - log_trace(correlation_id, f" Size: {payload.size} bytes") - log_trace(correlation_id, f" Encoding: {payload.encoding}") - - if payload.transport == "link": - log_trace(correlation_id, f" URL: {payload.data}") - - print(f"Test completed. Correlation ID: {correlation_id}") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test/test_micropython_text_receiver.py b/test/test_micropython_text_receiver.py deleted file mode 100644 index ee585d3..0000000 --- a/test/test_micropython_text_receiver.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for text transport testing - Receiver -Tests receiving text messages via NATS using nats_bridge.py smartreceive -""" - -import sys -import os -import json - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartreceive, log_trace -import nats -import asyncio - -# Configuration -SUBJECT = "/NATSBridge_text_test" -NATS_URL = "nats://nats.yiem.cc:4222" - - -async def main(): - log_trace("", f"Starting text transport receiver test...") - log_trace("", f"Note: This receiver will wait for messages from the sender.") - log_trace("", f"Run test_micropython_text_sender.py first to send test data.") - - # Connect to NATS - nc = await nats.connect(NATS_URL) - log_trace("", f"Connected to NATS at {NATS_URL}") - - # Subscribe to the subject - async def message_handler(msg): - log_trace("", f"Received message on {msg.subject}") - - # Use smartreceive to handle the data - result = smartreceive(msg.data) - - # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples - for dataname, data, data_type in result["payloads"]: - if isinstance(data, str): - log_trace(result.get("correlationId", ""), f"Received text '{dataname}' of type {data_type}") - log_trace(result.get("correlationId", ""), f" Length: {len(data)} characters") - - # Display first 100 characters - if len(data) > 100: - log_trace(result.get("correlationId", ""), f" First 100 characters: {data[:100]}...") - else: - log_trace(result.get("correlationId", ""), f" Content: {data}") - - # Save to file - output_path = f"./received_{dataname}.txt" - with open(output_path, 'w') as f: - f.write(data) - log_trace(result.get("correlationId", ""), f"Saved text to {output_path}") - else: - log_trace(result.get("correlationId", ""), f"Received unexpected data type for '{dataname}': {type(data)}") - - sid = await nc.subscribe(SUBJECT, cb=message_handler) - log_trace("", f"Subscribed to {SUBJECT} with subscription ID: {sid}") - - # Keep listening for 120 seconds - await asyncio.sleep(120) - await nc.close() - log_trace("", "Test completed.") - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/test/test_micropython_text_sender.py b/test/test_micropython_text_sender.py deleted file mode 100644 index 6aaf7f9..0000000 --- a/test/test_micropython_text_sender.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for text transport testing - Micropython -Tests sending text messages via NATS using nats_bridge.py smartsend -""" - -import sys -import os - -# Add src to path for import -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from nats_bridge import smartsend, log_trace -import uuid - -# Configuration -SUBJECT = "/NATSBridge_text_test" -NATS_URL = "nats://nats.yiem.cc:4222" -FILESERVER_URL = "http://192.168.88.104:8080" -SIZE_THRESHOLD = 1_000_000 # 1MB - -# Create correlation ID for tracing -correlation_id = str(uuid.uuid4()) - - -def main(): - # Create a small text (will use direct transport) - small_text = "Hello, this is a small text message. Testing direct transport via NATS." - - # Create a large text (will use link transport if > 1MB) - # Generate a larger text (~2MB to ensure link transport) - large_text = "\n".join([ - f"Line {i}: This is a sample text line with some content to pad the size. " * 100 - for i in range(500) - ]) - - # Test data 1: small text - data1 = ("small_text", small_text, "text") - - # Test data 2: large text - data2 = ("large_text", large_text, "text") - - log_trace(correlation_id, f"Starting smartsend for subject: {SUBJECT}") - log_trace(correlation_id, f"Correlation ID: {correlation_id}") - - # Use smartsend with text type - # For small text: will use direct transport (Base64 encoded UTF-8) - # For large text: will use link transport (uploaded to fileserver) - env, env_json_str = smartsend( - SUBJECT, - [data1, data2], # List of (dataname, data, type) tuples - nats_url=NATS_URL, - fileserver_url=FILESERVER_URL, - size_threshold=SIZE_THRESHOLD, - correlation_id=correlation_id, - msg_purpose="chat", - sender_name="text_sender", - receiver_name="", - receiver_id="", - reply_to="", - reply_to_msg_id="", - is_publish=True # Publish the message to NATS - ) - - log_trace(correlation_id, f"Sent message with {len(env.payloads)} payloads") - - # Log transport type for each payload - for i, payload in enumerate(env.payloads): - log_trace(correlation_id, f"Payload {i+1} ('{payload.dataname}'):") - log_trace(correlation_id, f" Transport: {payload.transport}") - log_trace(correlation_id, f" Type: {payload.type}") - log_trace(correlation_id, f" Size: {payload.size} bytes") - log_trace(correlation_id, f" Encoding: {payload.encoding}") - - if payload.transport == "link": - log_trace(correlation_id, f" URL: {payload.data}") - - print(f"Test completed. Correlation ID: {correlation_id}") - - -if __name__ == "__main__": - main() \ No newline at end of file -- 2.49.1 From fcc50847e4472f529ec075d1871175181525eb56 Mon Sep 17 00:00:00 2001 From: narawat Date: Wed, 25 Feb 2026 20:29:08 +0700 Subject: [PATCH 34/35] update --- src/NATSBridge.js | 753 --------------------------------------- src/nats_bridge.py | 871 --------------------------------------------- 2 files changed, 1624 deletions(-) delete mode 100644 src/NATSBridge.js delete mode 100644 src/nats_bridge.py diff --git a/src/NATSBridge.js b/src/NATSBridge.js deleted file mode 100644 index 2bb776d..0000000 --- a/src/NATSBridge.js +++ /dev/null @@ -1,753 +0,0 @@ -/** - * NATSBridge.js - Bi-Directional Data Bridge for JavaScript - * Implements smartsend and smartreceive for NATS communication - * - * This module provides functionality for sending and receiving data across network boundaries - * using NATS as the message bus, with support for both direct payload transport and - * URL-based transport for larger payloads. - * - * File Server Handler Architecture: - * The system uses handler functions to abstract file server operations, allowing support - * for different file server implementations (e.g., Plik, AWS S3, custom HTTP server). - * - * Handler Function Signatures: - * - * ```javascript - * // Upload handler - uploads data to file server and returns URL - * // The handler is passed to smartsend as fileserverUploadHandler parameter - * // It receives: (fileserver_url, dataname, data) - * // Returns: { status, uploadid, fileid, url } - * async function plik_oneshot_upload(fileserver_url, dataname, data) { ... } - * - * // Download handler - fetches data from file server URL with exponential backoff - * // The handler is passed to smartreceive as fileserverDownloadHandler parameter - * // It receives: (url, max_retries, base_delay, max_delay, correlation_id) - * // Returns: ArrayBuffer (the downloaded data) - * async function fileserverDownloadHandler(url, max_retries, base_delay, max_delay, correlation_id) { ... } - * ``` - * - * Multi-Payload Support (Standard API): - * The system uses a standardized list-of-tuples format for all payload operations. - * Even when sending a single payload, the user must wrap it in a list. - * - * API Standard: - * ```javascript - * // Input format for smartsend (always a list of tuples with type info) - * [{ dataname, data, type }, ...] - * - * // Output format for smartreceive (always returns a list of tuples) - * [{ dataname, data, type }, ...] - * ``` - * - * Supported types: "text", "dictionary", "table", "image", "audio", "video", "binary" - */ - -// ---------------------------------------------- 100 --------------------------------------------- # - -// Constants -const DEFAULT_SIZE_THRESHOLD = 1_000_000; // 1MB - threshold for switching from direct to link transport -const DEFAULT_NATS_URL = "nats://localhost:4222"; // Default NATS server URL -const DEFAULT_FILESERVER_URL = "http://localhost:8080"; // Default HTTP file server URL for link transport - -// Helper: Generate UUID v4 -function uuid4() { - // Simple UUID v4 generator - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); - return v.toString(16); - }); -} - -// Helper: Log with correlation ID and timestamp -function log_trace(correlation_id, message) { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); -} - -// Helper: Get size of data in bytes -function getDataSize(data) { - if (typeof data === 'string') { - return new TextEncoder().encode(data).length; - } else if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data.byteLength; - } else if (typeof data === 'object' && data !== null) { - // For objects, serialize to JSON and measure - return new TextEncoder().encode(JSON.stringify(data)).length; - } - return 0; -} - -// Helper: Convert ArrayBuffer to Base64 string -function arrayBufferToBase64(buffer) { - const bytes = new Uint8Array(buffer); - let binary = ''; - for (let i = 0; i < bytes.length; i++) { - binary += String.fromCharCode(bytes[i]); - } - return btoa(binary); -} - -// Helper: Convert Base64 string to ArrayBuffer -function base64ToArrayBuffer(base64) { - const binaryString = atob(base64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes.buffer; -} - -// Helper: Convert Uint8Array to Base64 string -function uint8ArrayToBase64(uint8array) { - let binary = ''; - for (let i = 0; i < uint8array.byteLength; i++) { - binary += String.fromCharCode(uint8array[i]); - } - return btoa(binary); -} - -// Helper: Convert Base64 string to Uint8Array -function base64ToUint8Array(base64) { - const binaryString = atob(base64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes; -} - -// Helper: Serialize data based on type -function _serialize_data(data, type) { - /** - * Serialize data according to specified format - * - * Supported formats: - * - "text": Treats data as text and converts to UTF-8 bytes - * - "dictionary": Serializes data as JSON and returns the UTF-8 byte representation - * - "table": Serializes data as an Arrow IPC stream (table format) - NOT IMPLEMENTED (requires arrow library) - * - "image": Expects binary data (ArrayBuffer) and returns it as bytes - * - "audio": Expects binary data (ArrayBuffer) and returns it as bytes - * - "video": Expects binary data (ArrayBuffer) and returns it as bytes - * - "binary": Generic binary data (ArrayBuffer or Uint8Array) and returns bytes - */ - if (type === "text") { - if (typeof data === 'string') { - return new TextEncoder().encode(data); - } else { - throw new Error("Text data must be a String"); - } - } else if (type === "dictionary") { - // JSON data - serialize directly - const jsonStr = JSON.stringify(data); - return new TextEncoder().encode(jsonStr); - } else if (type === "table") { - // Table data - convert to Arrow IPC stream (NOT IMPLEMENTED in pure JavaScript) - // This would require the apache-arrow library - throw new Error("Table serialization requires apache-arrow library"); - } else if (type === "image") { - if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? new Uint8Array(data) : data; - } else { - throw new Error("Image data must be ArrayBuffer or Uint8Array"); - } - } else if (type === "audio") { - if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? new Uint8Array(data) : data; - } else { - throw new Error("Audio data must be ArrayBuffer or Uint8Array"); - } - } else if (type === "video") { - if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? new Uint8Array(data) : data; - } else { - throw new Error("Video data must be ArrayBuffer or Uint8Array"); - } - } else if (type === "binary") { - if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? new Uint8Array(data) : data; - } else { - throw new Error("Binary data must be ArrayBuffer or Uint8Array"); - } - } else { - throw new Error(`Unknown type: ${type}`); - } -} - -// Helper: Deserialize bytes based on type -function _deserialize_data(data, type, correlation_id) { - /** - * Deserialize bytes to data based on type - * - * Supported formats: - * - "text": Converts bytes to string - * - "dictionary": Parses JSON string - * - "table": Parses Arrow IPC stream - NOT IMPLEMENTED (requires apache-arrow library) - * - "image": Returns binary data - * - "audio": Returns binary data - * - "video": Returns binary data - * - "binary": Returns binary data - */ - if (type === "text") { - const decoder = new TextDecoder(); - return decoder.decode(data); - } else if (type === "dictionary") { - const decoder = new TextDecoder(); - const jsonStr = decoder.decode(data); - return JSON.parse(jsonStr); - } else if (type === "table") { - // Table data - deserialize Arrow IPC stream (NOT IMPLEMENTED in pure JavaScript) - throw new Error("Table deserialization requires apache-arrow library"); - } else if (type === "image") { - return data; - } else if (type === "audio") { - return data; - } else if (type === "video") { - return data; - } else if (type === "binary") { - return data; - } else { - throw new Error(`Unknown type: ${type}`); - } -} - -// Helper: Upload data to file server -// Internal wrapper that adds correlation_id logging for smartsend -async function _upload_to_fileserver(fileserver_url, dataname, data, correlation_id) { - /** - * Internal upload helper - wraps plik_oneshot_upload to add correlation_id logging - * This allows smartsend to pass correlation_id for tracing without changing the handler signature - */ - log_trace(correlation_id, `Uploading ${dataname} to fileserver: ${fileserver_url}`); - const result = await plik_oneshot_upload(fileserver_url, dataname, data); - log_trace(correlation_id, `Uploaded to URL: ${result.url}`); - return result; -} - -// Helper: Fetch data from URL with exponential backoff -async function _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) { - /** - * Fetch data from URL with retry logic using exponential backoff - */ - let delay = base_delay; - - for (let attempt = 1; attempt <= max_retries; attempt++) { - try { - const response = await fetch(url); - - if (response.status === 200) { - log_trace(correlation_id, `Successfully fetched data from ${url} on attempt ${attempt}`); - const arrayBuffer = await response.arrayBuffer(); - return new Uint8Array(arrayBuffer); - } else { - throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`); - } - } catch (e) { - log_trace(correlation_id, `Attempt ${attempt} failed: ${e.message}`); - - if (attempt < max_retries) { - // Sleep with exponential backoff - await new Promise(resolve => setTimeout(resolve, delay)); - delay = Math.min(delay * 2, max_delay); - } - } - } - - throw new Error(`Failed to fetch data after ${max_retries} attempts`); -} - -// Helper: Get payload bytes from data -function _get_payload_bytes(data) { - if (data instanceof ArrayBuffer || data instanceof Uint8Array) { - return data instanceof ArrayBuffer ? new Uint8Array(data) : data; - } else if (typeof data === 'string') { - return new TextEncoder().encode(data); - } else { - // For objects, serialize to JSON - return new TextEncoder().encode(JSON.stringify(data)); - } -} - -// MessagePayload class - matches msg_payload_v1 Julia struct -class MessagePayload { - /** - * Represents a single payload in the message envelope - * Matches Julia's msg_payload_v1 struct - * - * @param {Object} options - Payload options - * @param {string} options.id - ID of this payload (e.g., "uuid4") - * @param {string} options.dataname - Name of this payload (e.g., "login_image") - * @param {string} options.payload_type - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" - * @param {string} options.transport - "direct" or "link" - * @param {string} options.encoding - "none", "json", "base64", "arrow-ipc" - * @param {number} options.size - Data size in bytes - * @param {string|Uint8Array} options.data - Payload data (Uint8Array for direct, URL string for link) - * @param {Object} options.metadata - Metadata for this payload - */ - constructor(options) { - this.id = options.id || uuid4(); - this.dataname = options.dataname; - this.payload_type = options.payload_type; - this.transport = options.transport; - this.encoding = options.encoding; - this.size = options.size; - this.data = options.data; - this.metadata = options.metadata || {}; - } - - // Convert to JSON object - uses snake_case to match Julia API - toJSON() { - const obj = { - id: this.id, - dataname: this.dataname, - payload_type: this.payload_type, - transport: this.transport, - encoding: this.encoding, - size: this.size - }; - - // Include data based on transport type - if (this.transport === "direct" && this.data !== null && this.data !== undefined) { - if (this.encoding === "base64" || this.encoding === "json") { - obj.data = this.data; - } else { - // For other encodings, use base64 - const payloadBytes = _get_payload_bytes(this.data); - obj.data = uint8ArrayToBase64(payloadBytes); - } - } else if (this.transport === "link" && this.data !== null && this.data !== undefined) { - // For link transport, data is a URL string - obj.data = this.data; - } - - if (Object.keys(this.metadata).length > 0) { - obj.metadata = this.metadata; - } - - return obj; - } -} - -// MessageEnvelope class - matches msg_envelope_v1 Julia struct -class MessageEnvelope { - /** - * Represents the message envelope containing metadata and payloads - * Matches Julia's msg_envelope_v1 struct - * - * @param {Object} options - Envelope options - * @param {string} options.correlation_id - Unique identifier to track messages - * @param {string} options.msg_id - This message id - * @param {string} options.timestamp - Message published timestamp - * @param {string} options.send_to - Topic/subject the sender sends to - * @param {string} options.msg_purpose - Purpose of this message - * @param {string} options.sender_name - Name of the sender - * @param {string} options.sender_id - UUID of the sender - * @param {string} options.receiver_name - Name of the receiver - * @param {string} options.receiver_id - UUID of the receiver - * @param {string} options.reply_to - Topic to reply to - * @param {string} options.reply_to_msg_id - Message id this message is replying to - * @param {string} options.broker_url - NATS server address - * @param {Object} options.metadata - Metadata for the envelope - * @param {Array} options.payloads - Array of payloads - */ - constructor(options) { - this.correlation_id = options.correlation_id || uuid4(); - this.msg_id = options.msg_id || uuid4(); - this.timestamp = options.timestamp || new Date().toISOString(); - this.send_to = options.send_to; - this.msg_purpose = options.msg_purpose || ""; - this.sender_name = options.sender_name || ""; - this.sender_id = options.sender_id || uuid4(); - this.receiver_name = options.receiver_name || ""; - this.receiver_id = options.receiver_id || ""; - this.reply_to = options.reply_to || ""; - this.reply_to_msg_id = options.reply_to_msg_id || ""; - this.broker_url = options.broker_url || DEFAULT_NATS_URL; - this.metadata = options.metadata || {}; - this.payloads = options.payloads || []; - } - - // Convert to JSON object - uses snake_case to match Julia API - toJSON() { - const obj = { - correlation_id: this.correlation_id, - msg_id: this.msg_id, - timestamp: this.timestamp, - send_to: this.send_to, - msg_purpose: this.msg_purpose, - sender_name: this.sender_name, - sender_id: this.sender_id, - receiver_name: this.receiver_name, - receiver_id: this.receiver_id, - reply_to: this.reply_to, - reply_to_msg_id: this.reply_to_msg_id, - broker_url: this.broker_url - }; - - if (Object.keys(this.metadata).length > 0) { - obj.metadata = this.metadata; - } - - if (this.payloads.length > 0) { - obj.payloads = this.payloads.map(p => p.toJSON()); - } - - return obj; - } - - // Convert to JSON string - toString() { - return JSON.stringify(this.toJSON()); - } -} - -// SmartSend function - matches Julia smartsend signature and behavior -async function smartsend(subject, data, options = {}) { - /** - * Send data either directly via NATS or via a fileserver URL, depending on payload size - * - * This function intelligently routes data delivery based on payload size relative to a threshold. - * If the serialized payload is smaller than `size_threshold`, it encodes the data as Base64 and publishes directly over NATS. - * Otherwise, it uploads the data to a fileserver and publishes only the download URL over NATS. - * - * @param {string} subject - NATS subject to publish the message to - * @param {Array} data - List of {dataname, data, type} objects to send (must be a list, even for single payload) - * @param {Object} options - Additional options - * @param {string} options.broker_url - URL of the NATS server (default: "nats://localhost:4222") - * @param {string} options.fileserver_url - Base URL of the file server (default: "http://localhost:8080") - * @param {Function} options.fileserver_upload_handler - Function to handle fileserver uploads - * @param {number} options.size_threshold - Threshold in bytes separating direct vs link transport (default: 1MB) - * @param {string} options.correlation_id - Optional correlation ID for tracing - * @param {string} options.msg_purpose - Purpose of the message (default: "chat") - * @param {string} options.sender_name - Name of the sender (default: "NATSBridge") - * @param {string} options.receiver_name - Name of the receiver (default: "") - * @param {string} options.receiver_id - UUID of the receiver (default: "") - * @param {string} options.reply_to - Topic to reply to (default: "") - * @param {string} options.reply_to_msg_id - Message ID this message is replying to (default: "") - * @param {boolean} options.is_publish - Whether to automatically publish the message to NATS (default: true) - * - When true: Message is published to NATS automatically - * - When false: Returns (env, env_json_str) without publishing, allowing manual publishing - * @returns {Promise} - A tuple-like object with { env: MessageEnvelope, env_json_str: string } - * - env: MessageEnvelope object with all metadata and payloads - * - env_json_str: JSON string representation of the envelope for manual publishing - */ - const { - broker_url = DEFAULT_NATS_URL, - fileserver_url = DEFAULT_FILESERVER_URL, - fileserver_upload_handler = _upload_to_fileserver, - size_threshold = DEFAULT_SIZE_THRESHOLD, - correlation_id = uuid4(), - msg_purpose = "chat", - sender_name = "NATSBridge", - receiver_name = "", - receiver_id = "", - reply_to = "", - reply_to_msg_id = "", - is_publish = true // Whether to automatically publish the message to NATS - } = options; - - log_trace(correlation_id, `Starting smartsend for subject: ${subject}`); - - // Generate message metadata - const msg_id = uuid4(); - - // Process each payload in the list - const payloads = []; - - for (const payload of data) { - const dataname = payload.dataname; - const payloadData = payload.data; - const payloadType = payload.type; - - // Serialize data based on type - const payloadBytes = _serialize_data(payloadData, payloadType); - const payloadSize = payloadBytes.byteLength; - - log_trace(correlation_id, `Serialized payload '${dataname}' (payload_type: ${payloadType}) size: ${payloadSize} bytes`); - - // Decision: Direct vs Link - if (payloadSize < size_threshold) { - // Direct path - Base64 encode and send via NATS - const payloadB64 = uint8ArrayToBase64(payloadBytes); - log_trace(correlation_id, `Using direct transport for ${payloadSize} bytes`); - - // Create MessagePayload for direct transport - const payloadObj = new MessagePayload({ - dataname: dataname, - payload_type: payloadType, - transport: "direct", - encoding: "base64", - size: payloadSize, - data: payloadB64, - metadata: { payload_bytes: payloadSize } - }); - payloads.push(payloadObj); - } else { - // Link path - Upload to HTTP server, send URL via NATS - log_trace(correlation_id, `Using link transport, uploading to fileserver`); - - // Upload to HTTP server using plik_oneshot_upload handler - const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes); - - if (response.status !== 200) { - throw new Error(`Failed to upload data to fileserver: ${response.status}`); - } - - const url = response.url; - log_trace(correlation_id, `Uploaded to URL: ${url}`); - - // Create MessagePayload for link transport - const payloadObj = new MessagePayload({ - dataname: dataname, - payload_type: payloadType, - transport: "link", - encoding: "none", - size: payloadSize, - data: url, - metadata: {} - }); - payloads.push(payloadObj); - } - } - - // Create MessageEnvelope with all payloads - const env = new MessageEnvelope({ - correlation_id: correlation_id, - msg_id: msg_id, - send_to: subject, - msg_purpose: msg_purpose, - sender_name: sender_name, - receiver_name: receiver_name, - receiver_id: receiver_id, - reply_to: reply_to, - reply_to_msg_id: reply_to_msg_id, - broker_url: broker_url, - payloads: payloads - }); - - // Convert envelope to JSON string - const env_json_str = env.toString(); - - // Publish to NATS if isPublish is true - if (is_publish) { - await publish_message(broker_url, subject, env_json_str, correlation_id); - } - - // Return both envelope and JSON string (tuple-like structure, matching Julia API) - return { - env: env, - env_json_str: env_json_str - }; -} - -// Helper: Publish message to NATS -async function publish_message(broker_url, subject, message, correlation_id) { - /** - * Publish a message to a NATS subject with proper connection management - * - * @param {string} broker_url - NATS server URL - * @param {string} subject - NATS subject to publish to - * @param {string} message - JSON message to publish - * @param {string} correlation_id - Correlation ID for logging - */ - log_trace(correlation_id, `Publishing message to ${subject}`); - - // For Node.js, we would use nats.js library - // This is a placeholder that throws an error - // In production, you would import and use the actual nats library - - // Example with nats.js: - // import { connect } from 'nats'; - // const nc = await connect({ servers: [broker_url] }); - // await nc.publish(subject, message); - // nc.close(); - - // For now, just log the message - console.log(`[NATS PUBLISH] Subject: ${subject}, Message: ${message.substring(0, 100)}...`); -} - -// SmartReceive function - matches Julia smartreceive signature and behavior -async function smartreceive(msg, options = {}) { - /** - * Receive and process messages from NATS - * - * This function processes incoming NATS messages, handling both direct transport - * (base64 decoded payloads) and link transport (URL-based payloads). - * - * @param {Object} msg - NATS message object with payload property - * @param {Object} options - Additional options - * @param {Function} options.fileserver_download_handler - Function to handle downloading data from file server URLs - * @param {number} options.max_retries - Maximum retry attempts for fetching URL (default: 5) - * @param {number} options.base_delay - Initial delay for exponential backoff in ms (default: 100) - * @param {number} options.max_delay - Maximum delay for exponential backoff in ms (default: 5000) - * - * @returns {Promise} - JSON object of envelope with payloads field containing list of {dataname, data, type} tuples - */ - const { - fileserver_download_handler = _fetch_with_backoff, - max_retries = 5, - base_delay = 100, - max_delay = 5000 - } = options; - - // Parse the JSON envelope - const jsonStr = typeof msg.payload === 'string' ? msg.payload : new TextDecoder().decode(msg.payload); - const json_data = JSON.parse(jsonStr); - - log_trace(json_data.correlation_id, `Processing received message`); - - // Process all payloads in the envelope - const payloads_list = []; - - // Get number of payloads - const num_payloads = json_data.payloads ? json_data.payloads.length : 0; - - for (let i = 0; i < num_payloads; i++) { - const payload = json_data.payloads[i]; - const transport = payload.transport; - const dataname = payload.dataname; - - if (transport === "direct") { - // Direct transport - payload is in the message - log_trace(json_data.correlation_id, `Direct transport - decoding payload '${dataname}'`); - - // Extract base64 payload from the payload - const payload_b64 = payload.data; - - // Decode Base64 payload - const payload_bytes = base64ToUint8Array(payload_b64); - - // Deserialize based on type - const data_type = payload.payload_type; - const data = _deserialize_data(payload_bytes, data_type, json_data.correlation_id); - - payloads_list.push({ dataname, data, type: data_type }); - } else if (transport === "link") { - // Link transport - payload is at URL - const url = payload.data; - log_trace(json_data.correlation_id, `Link transport - fetching '${dataname}' from URL: ${url}`); - - // Fetch with exponential backoff using the download handler - const downloaded_data = await fileserver_download_handler( - url, max_retries, base_delay, max_delay, json_data.correlation_id - ); - - // Deserialize based on type - const data_type = payload.payload_type; - const data = _deserialize_data(downloaded_data, data_type, json_data.correlation_id); - - payloads_list.push({ dataname, data, type: data_type }); - } else { - throw new Error(`Unknown transport type for payload '${dataname}': ${transport}`); - } - } - - // Replace payloads array with the processed list of {dataname, data, type} tuples - // This matches Julia's smartreceive return format - json_data.payloads = payloads_list; - - return json_data; -} - -// plik_oneshot_upload - matches Julia plik_oneshot_upload function -// Upload handler signature: plik_oneshot_upload(fileserver_url, dataname, data) -// Returns: { status, uploadid, fileid, url } -async function plik_oneshot_upload(file_server_url, dataname, data) { - /** - * Upload a single file to a plik server using one-shot mode - * This function uploads raw byte array to a plik server in one-shot mode (no upload session). - * It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, - * retrieves an upload ID and token, then uploads the file data as multipart form data using the token. - * - * This is the default upload handler used by smartsend. - * Custom handlers can be passed via the fileserver_upload_handler option. - * - * @param {string} file_server_url - Base URL of the plik server (e.g., "http://localhost:8080") - * @param {string} dataname - Name of the file being uploaded - * @param {Uint8Array} data - Raw byte data of the file content - * @returns {Promise} - Dictionary with keys: status, uploadid, fileid, url - */ - - // Step 1: Get upload ID and token - const url_getUploadID = `${file_server_url}/upload`; - const headers = { "Content-Type": "application/json" }; - const body = JSON.stringify({ OneShot: true }); - - let http_response = await fetch(url_getUploadID, { - method: "POST", - headers: headers, - body: body - }); - - const response_json = await http_response.json(); - const uploadid = response_json.id; - const uploadtoken = response_json.uploadToken; - - // Step 2: Upload file data - const url_upload = `${file_server_url}/file/${uploadid}`; - - // Create multipart form data - const formData = new FormData(); - const blob = new Blob([data], { type: "application/octet-stream" }); - formData.append("file", blob, dataname); - - http_response = await fetch(url_upload, { - method: "POST", - headers: { "X-UploadToken": uploadtoken }, - body: formData - }); - - const fileResponseJson = await http_response.json(); - const fileid = fileResponseJson.id; - - // URL of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" - const url = `${file_server_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; - - return { - status: http_response.status, - uploadid: uploadid, - fileid: fileid, - url: url - }; -} - -// Export for Node.js -if (typeof module !== 'undefined' && module.exports) { - module.exports = { - MessageEnvelope, - MessagePayload, - smartsend, - smartreceive, - _serialize_data, - _deserialize_data, - _fetch_with_backoff, - _upload_to_fileserver, - plik_oneshot_upload, - DEFAULT_SIZE_THRESHOLD, - DEFAULT_NATS_URL, - DEFAULT_FILESERVER_URL, - uuid4, - log_trace - }; -} - -// Export for browser -if (typeof window !== 'undefined') { - window.NATSBridge = { - MessageEnvelope, - MessagePayload, - smartsend, - smartreceive, - _serialize_data, - _deserialize_data, - _fetch_with_backoff, - _upload_to_fileserver, - plik_oneshot_upload, - DEFAULT_SIZE_THRESHOLD, - DEFAULT_NATS_URL, - DEFAULT_FILESERVER_URL, - uuid4, - log_trace - }; -} \ No newline at end of file diff --git a/src/nats_bridge.py b/src/nats_bridge.py deleted file mode 100644 index ef2b726..0000000 --- a/src/nats_bridge.py +++ /dev/null @@ -1,871 +0,0 @@ -""" -Python NATS Bridge - Bi-Directional Data Bridge - -This module provides functionality for sending and receiving data over NATS -using the Claim-Check pattern for large payloads. - -Supported types: "text", "dictionary", "table", "image", "audio", "video", "binary" - -Multi-Payload Support (Standard API): -The system uses a standardized list-of-tuples format for all payload operations. -Even when sending a single payload, the user must wrap it in a list. - -API Standard: - # Input format for smartsend (always a list of tuples with type info) - [(dataname1, data1, type1), (dataname2, data2, type2), ...] - - # Output format for smartreceive (returns a dictionary with payloads field containing list of tuples) - # Returns: Dict with envelope metadata and payloads field containing list of tuples - # { - # "correlation_id": "...", - # "msg_id": "...", - # "timestamp": "...", - # "send_to": "...", - # "msg_purpose": "...", - # "sender_name": "...", - # "sender_id": "...", - # "receiver_name": "...", - # "receiver_id": "...", - # "reply_to": "...", - # "reply_to_msg_id": "...", - # "broker_url": "...", - # "metadata": {...}, - # "payloads": [(dataname1, data1, type1), (dataname2, data2, type2), ...] - # } -""" - -import json -import time -import uuid - -# Constants -DEFAULT_SIZE_THRESHOLD = 1000000 # 1MB - threshold for switching from direct to link transport -DEFAULT_BROKER_URL = "nats://localhost:4222" -DEFAULT_FILESERVER_URL = "http://localhost:8080" - -# ============================================= 100 ============================================== # - - -class MessagePayload: - """Internal message payload structure representing a single payload within a NATS message envelope. - - This structure represents a single payload within a NATS message envelope. - It supports both direct transport (base64-encoded data) and link transport (URL-based). - - Attributes: - id: Unique identifier for this payload (e.g., "uuid4") - dataname: Name of the payload (e.g., "login_image") - payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") - transport: Transport method ("direct" or "link") - encoding: Encoding method ("none", "json", "base64", "arrow-ipc") - size: Size of the payload in bytes - data: Payload data (bytes for direct, URL for link) - metadata: Optional metadata dictionary - """ - - def __init__(self, data, payload_type, id="", dataname="", transport="direct", - encoding="none", size=0, metadata=None): - """ - Initialize a MessagePayload. - - Args: - data: Payload data (base64 string for direct, URL string for link) - payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") - id: Unique identifier for this payload (auto-generated if empty) - dataname: Name of the payload (auto-generated UUID if empty) - transport: Transport method ("direct" or "link") - encoding: Encoding method ("none", "json", "base64", "arrow-ipc") - size: Size of the payload in bytes - metadata: Optional metadata dictionary - """ - self.id = id if id else self._generate_uuid() - self.dataname = dataname if dataname else self._generate_uuid() - self.payload_type = payload_type - self.transport = transport - self.encoding = encoding - self.size = size - self.data = data - self.metadata = metadata if metadata else {} - - def _generate_uuid(self): - """Generate a UUID string.""" - return str(uuid.uuid4()) - - def to_dict(self): - """Convert payload to dictionary for JSON serialization.""" - payload_dict = { - "id": self.id, - "dataname": self.dataname, - "payload_type": self.payload_type, - "transport": self.transport, - "encoding": self.encoding, - "size": self.size, - } - - # Include data based on transport type - if self.transport == "direct" and self.data is not None: - if self.encoding == "base64" or self.encoding == "json": - payload_dict["data"] = self.data - else: - # For other encodings, use base64 - payload_dict["data"] = self._to_base64(self.data) - elif self.transport == "link" and self.data is not None: - # For link transport, data is a URL string - payload_dict["data"] = self.data - - if self.metadata: - payload_dict["metadata"] = self.metadata - - return payload_dict - - def _to_base64(self, data): - """Convert bytes to base64 string.""" - if isinstance(data, bytes): - # Simple base64 encoding without library - import ubinascii - return ubinascii.b2a_base64(data).decode('utf-8').strip() - return data - - def _from_base64(self, data): - """Convert base64 string to bytes.""" - import ubinascii - return ubinascii.a2b_base64(data) - - -class MessageEnvelope: - """Internal message envelope structure containing multiple payloads with metadata.""" - - def __init__(self, send_to, payloads, correlation_id="", msg_id="", timestamp="", - msg_purpose="", sender_name="", sender_id="", receiver_name="", - receiver_id="", reply_to="", reply_to_msg_id="", broker_url=DEFAULT_BROKER_URL, - metadata=None): - """ - Initialize a MessageEnvelope. - - Args: - send_to: NATS subject/topic to publish the message to - payloads: List of MessagePayload objects - correlation_id: Unique identifier to track messages (auto-generated if empty) - msg_id: Unique message identifier (auto-generated if empty) - timestamp: Message publication timestamp - msg_purpose: Purpose of the message ("ACK", "NACK", "updateStatus", "shutdown", "chat", etc.) - sender_name: Name of the sender - sender_id: UUID of the sender - receiver_name: Name of the receiver (empty means broadcast) - receiver_id: UUID of the receiver (empty means broadcast) - reply_to: Topic where receiver should reply - reply_to_msg_id: Message ID this message is replying to - broker_url: NATS broker URL - metadata: Optional message-level metadata - """ - self.correlation_id = correlation_id if correlation_id else self._generate_uuid() - self.msg_id = msg_id if msg_id else self._generate_uuid() - self.timestamp = timestamp if timestamp else self._get_timestamp() - self.send_to = send_to - self.msg_purpose = msg_purpose - self.sender_name = sender_name - self.sender_id = sender_id if sender_id else self._generate_uuid() - self.receiver_name = receiver_name - self.receiver_id = receiver_id if receiver_id else self._generate_uuid() - self.reply_to = reply_to - self.reply_to_msg_id = reply_to_msg_id - self.broker_url = broker_url - self.metadata = metadata if metadata else {} - self.payloads = payloads - - def _generate_uuid(self): - """Generate a UUID string.""" - return str(uuid.uuid4()) - - def _get_timestamp(self): - """Get current timestamp in ISO format.""" - # Simplified timestamp - Micropython may not have full datetime - return "2026-02-21T" + time.strftime("%H:%M:%S", time.localtime()) - - def to_json(self): - """Convert envelope to JSON string. - - Returns: - str: JSON string representation of the envelope using snake_case field names - """ - obj = { - "correlation_id": self.correlation_id, - "msg_id": self.msg_id, - "timestamp": self.timestamp, - "send_to": self.send_to, - "msg_purpose": self.msg_purpose, - "sender_name": self.sender_name, - "sender_id": self.sender_id, - "receiver_name": self.receiver_name, - "receiver_id": self.receiver_id, - "reply_to": self.reply_to, - "reply_to_msg_id": self.reply_to_msg_id, - "broker_url": self.broker_url - } - - # Include metadata if not empty - if self.metadata: - obj["metadata"] = self.metadata - - # Convert payloads to JSON array - if self.payloads: - payloads_json = [] - for payload in self.payloads: - payloads_json.append(payload.to_dict()) - obj["payloads"] = payloads_json - - return json.dumps(obj) - - -def log_trace(correlation_id, message): - """Log a trace message with correlation ID and timestamp.""" - timestamp = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) - print("[{}] [Correlation: {}] {}".format(timestamp, correlation_id, message)) - - -def _serialize_data(data, payload_type): - """Serialize data according to specified format. - - This function serializes arbitrary data into a binary representation based on the specified type. - It supports multiple serialization formats for different data types. - - Args: - data: Data to serialize - - "text": String - - "dictionary": JSON-serializable dict - - "table": Tabular data (pandas DataFrame or list of dicts) - - "image", "audio", "video", "binary": bytes - payload_type: Target format ("text", "dictionary", "table", "image", "audio", "video", "binary") - - Returns: - bytes: Binary representation of the serialized data - - Example: - >>> text_bytes = _serialize_data("Hello World", "text") - >>> json_bytes = _serialize_data({"key": "value"}, "dictionary") - >>> table_bytes = _serialize_data([{"id": 1, "name": "Alice"}], "table") - """ - if payload_type == "text": - if isinstance(data, str): - return data.encode('utf-8') - else: - raise ValueError("Text data must be a string") - - elif payload_type == "dictionary": - if isinstance(data, dict): - json_str = json.dumps(data) - return json_str.encode('utf-8') - else: - raise ValueError("Dictionary data must be a dict") - - elif payload_type == "table": - # Support pandas DataFrame or list of dicts - try: - import pandas as pd - if isinstance(data, pd.DataFrame): - # Convert DataFrame to JSON and then to bytes - json_str = data.to_json(orient='records', force_ascii=False) - return json_str.encode('utf-8') - elif isinstance(data, list) and len(data) > 0 and isinstance(data[0], dict): - # List of dicts - json_str = json.dumps(data) - return json_str.encode('utf-8') - else: - raise ValueError("Table data must be a pandas DataFrame or list of dicts") - except ImportError: - # Fallback: if pandas not available, treat as list of dicts - if isinstance(data, list): - json_str = json.dumps(data) - return json_str.encode('utf-8') - else: - raise ValueError("Table data requires pandas DataFrame or list of dicts (pandas not available)") - - elif payload_type in ("image", "audio", "video", "binary"): - if isinstance(data, bytes): - return data - else: - raise ValueError("{} data must be bytes".format(payload_type.capitalize())) - - else: - raise ValueError("Unknown payload_type: {}".format(payload_type)) - - -def _deserialize_data(data_bytes, payload_type, correlation_id): - """Deserialize bytes to data based on type. - - This function converts serialized bytes back to Python data based on type. - It handles "text" (string), "dictionary" (JSON deserialization), "table" (JSON deserialization), - "image" (binary data), "audio" (binary data), "video" (binary data), and "binary" (binary data). - - Args: - data_bytes: Serialized data as bytes - payload_type: Data type ("text", "dictionary", "table", "image", "audio", "video", "binary") - correlation_id: Correlation ID for logging - - Returns: - Deserialized data: - - "text": str - - "dictionary": dict - - "table": list of dicts (or pandas DataFrame if available) - - "image", "audio", "video", "binary": bytes - - Example: - >>> text_data = _deserialize_data(b"Hello", "text", "corr_id") - >>> json_data = _deserialize_data(b'{"key": "value"}', "dictionary", "corr_id") - >>> table_data = _deserialize_data(b'[{"id": 1}]', "table", "corr_id") - """ - if payload_type == "text": - return data_bytes.decode('utf-8') - - elif payload_type == "dictionary": - json_str = data_bytes.decode('utf-8') - return json.loads(json_str) - - elif payload_type == "table": - # Deserialize table data (JSON format) - json_str = data_bytes.decode('utf-8') - table_data = json.loads(json_str) - # If pandas is available, try to convert to DataFrame - try: - import pandas as pd - return pd.DataFrame(table_data) - except ImportError: - return table_data - - elif payload_type in ("image", "audio", "video", "binary"): - return data_bytes - - else: - raise ValueError("Unknown payload_type: {}".format(payload_type)) - - -class NATSConnection: - """Simple NATS connection for Python and Micropython.""" - - def __init__(self, url=DEFAULT_BROKER_URL): - """Initialize NATS connection. - - Args: - url: NATS server URL (e.g., "nats://localhost:4222") - """ - self.url = url - self.host = "localhost" - self.port = 4222 - self.conn = None - self._parse_url(url) - - def _parse_url(self, url): - """Parse NATS URL to extract host and port.""" - if url.startswith("nats://"): - url = url[7:] - elif url.startswith("tls://"): - url = url[6:] - - if ":" in url: - self.host, port_str = url.split(":") - self.port = int(port_str) - else: - self.host = url - - def connect(self): - """Connect to NATS server.""" - # Use socket for both Python and Micropython - try: - import socket - addr = socket.getaddrinfo(self.host, self.port)[0][-1] - self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.conn.connect(addr) - except NameError: - # Micropython fallback - import usocket - addr = usocket.getaddrinfo(self.host, self.port)[0][-1] - self.conn = usocket.socket() - self.conn.connect(addr) - - log_trace("", "Connected to NATS server at {}:{}".format(self.host, self.port)) - - def publish(self, subject, message): - """Publish a message to a NATS subject. - - Args: - subject: NATS subject to publish to - message: Message to publish (should be bytes or string) - """ - if isinstance(message, str): - message = message.encode('utf-8') - - # Simple NATS protocol implementation - msg = "PUB {} {}\r\n".format(subject, len(message)) - msg = msg.encode('utf-8') + message + b"\r\n" - - try: - import socket - self.conn.send(msg) - except NameError: - # Micropython fallback - import usocket - self.conn.send(msg) - - log_trace("", "Message published to {}".format(subject)) - - def subscribe(self, subject, callback): - """Subscribe to a NATS subject. - - Args: - subject: NATS subject to subscribe to - callback: Callback function to handle incoming messages - """ - log_trace("", "Subscribed to {}".format(subject)) - # Simplified subscription - in a real implementation, you'd handle SUB/PUB messages - # For Micropython, we'll use a simple polling approach - self.subscribed_subject = subject - self.subscription_callback = callback - - def wait_message(self, timeout=1000): - """Wait for incoming message. - - Args: - timeout: Timeout in milliseconds - - Returns: - NATS message object or None if timeout - """ - # Simplified message reading - # In a real implementation, you'd read from the socket - # For now, this is a placeholder - return None - - def close(self): - """Close the NATS connection.""" - if self.conn: - self.conn.close() - self.conn = None - log_trace("", "NATS connection closed") - - -def _fetch_with_backoff(url, max_retries=5, base_delay=100, max_delay=5000, correlation_id=""): - """Fetch data from URL with exponential backoff. - - This function retrieves data from a URL with retry logic using - exponential backoff to handle transient failures. - - Args: - url: URL to fetch from - max_retries: Maximum number of retry attempts (default: 5) - base_delay: Initial delay in milliseconds (default: 100) - max_delay: Maximum delay in milliseconds (default: 5000) - correlation_id: Correlation ID for logging - - Returns: - bytes: Fetched data - - Raises: - Exception: If all retry attempts fail - - Example: - >>> data = _fetch_with_backoff("http://example.com/file.zip", 5, 100, 5000, "corr_id") - """ - delay = base_delay - for attempt in range(1, max_retries + 1): - try: - # Simple HTTP GET request - # Try urequests for Micropython first, then requests for Python - try: - import urequests - response = urequests.get(url) - status_code = response.status_code - content = response.content - except ImportError: - try: - import requests - response = requests.get(url) - response.raise_for_status() - status_code = response.status_code - content = response.content - except ImportError: - raise Exception("No HTTP library available (urequests or requests)") - - if status_code == 200: - log_trace(correlation_id, "Successfully fetched data from {} on attempt {}".format(url, attempt)) - return content - else: - raise Exception("Failed to fetch: {}".format(status_code)) - except Exception as e: - log_trace(correlation_id, "Attempt {} failed: {}".format(attempt, str(e))) - if attempt < max_retries: - time.sleep(delay / 1000.0) - delay = min(delay * 2, max_delay) - - raise Exception("Failed to fetch data after {} attempts".format(max_retries)) - - -def plik_oneshot_upload(fileserver_url, dataname, data): - """Upload a single file to a plik server using one-shot mode. - - This function uploads raw byte data to a plik server in one-shot mode (no upload session). - It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, - retrieves an upload ID and token, then uploads the file data as multipart form data using the token. - - Args: - fileserver_url: Base URL of the plik server (e.g., "http://localhost:8080") - dataname: Name of the file being uploaded - data: Raw byte data of the file content - - Returns: - dict: Dictionary with keys: - - "status": HTTP server response status - - "uploadid": ID of the one-shot upload session - - "fileid": ID of the uploaded file within the session - - "url": Full URL to download the uploaded file - - Example: - >>> result = plik_oneshot_upload("http://localhost:8080", "test.txt", b"hello world") - >>> result["status"], result["uploadid"], result["fileid"], result["url"] - """ - import json - - try: - import urequests - except ImportError: - import requests as urequests - - # Get upload ID - url_get_upload_id = "{}/upload".format(fileserver_url) - headers = {"Content-Type": "application/json"} - body = json.dumps({"OneShot": True}) - - response = urequests.post(url_get_upload_id, headers=headers, data=body) - response_json = json.loads(response.text if hasattr(response, 'text') else response.content) - - uploadid = response_json.get("id") - uploadtoken = response_json.get("uploadToken") - - # Upload file - url_upload = "{}/file/{}".format(fileserver_url, uploadid) - headers = {"X-UploadToken": uploadtoken} - - # For Micropython, we need to construct the multipart form data manually - # This is a simplified approach - boundary = "----WebKitFormBoundary{}".format(uuid.uuid4().hex[:16]) - - # Create multipart body - part1 = "--{}\r\n".format(boundary) - part1 += "Content-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n".format(dataname) - part1 += "Content-Type: application/octet-stream\r\n\r\n" - part1_bytes = part1.encode('utf-8') - - part2 = "\r\n--{}--".format(boundary) - part2_bytes = part2.encode('utf-8') - - # Combine all parts - full_body = part1_bytes + data + part2_bytes - - # Set content type with boundary - content_type = "multipart/form-data; boundary={}".format(boundary) - - response = urequests.post(url_upload, headers={"Content-Type": content_type}, data=full_body) - response_json = json.loads(response.text if hasattr(response, 'text') else response.content) - - fileid = response_json.get("id") - url = "{}/file/{}/{}".format(fileserver_url, uploadid, dataname) - - return { - "status": response.status_code, - "uploadid": uploadid, - "fileid": fileid, - "url": url - } - - -def smartsend(subject, data, broker_url=DEFAULT_BROKER_URL, fileserver_url=DEFAULT_FILESERVER_URL, - fileserver_upload_handler=plik_oneshot_upload, size_threshold=DEFAULT_SIZE_THRESHOLD, - correlation_id=None, msg_purpose="chat", sender_name="NATSBridge", - receiver_name="", receiver_id="", reply_to="", reply_to_msg_id="", is_publish=True): - """Send data either directly via NATS or via a fileserver URL, depending on payload size. - - This function intelligently routes data delivery based on payload size relative to a threshold. - If the serialized payload is smaller than `size_threshold`, it encodes the data as Base64 and - publishes directly over NATS. Otherwise, it uploads the data to a fileserver and publishes - only the download URL over NATS. - - API Standard: - - Input format: List of (dataname, data, payload_type) tuples - - Even single payloads must be wrapped in a list - - Each payload can have a different type, enabling mixed-content messages - - Args: - subject: NATS subject to publish the message to - data: List of (dataname, data, payload_type) tuples to send - - dataname: Name of the payload - - data: The actual data to send - - payload_type: Payload type ("text", "dictionary", "table", "image", "audio", "video", "binary") - - Example: [("message", "Hello World!", "text"), ("config", {"key": "value"}, "dictionary")] - broker_url: URL of the NATS server - fileserver_url: URL of the HTTP file server - fileserver_upload_handler: Function to handle fileserver uploads (must return dict with "status", "uploadid", "fileid", "url" keys) - size_threshold: Threshold in bytes separating direct vs link transport (default: 1MB) - correlation_id: Optional correlation ID for tracing; if None, a UUID is generated - msg_purpose: Purpose of the message ("ACK", "NACK", "updateStatus", "shutdown", "chat", etc.) - sender_name: Name of the sender - receiver_name: Name of the receiver (empty string means broadcast) - receiver_id: UUID of the receiver (empty string means broadcast) - reply_to: Topic to reply to (empty string if no reply expected) - reply_to_msg_id: Message ID this message is replying to - is_publish: Whether to automatically publish the message to NATS (default: True) - - When True: message is published to NATS - - When False: returns envelope and JSON string without publishing - - Returns: - tuple: (env, env_json_str) where: - - env: MessageEnvelope object with all metadata and payloads - - env_json_str: JSON string representation of the envelope for publishing - - Example: - >>> data = [("message", "Hello World!", "text")] - >>> env, env_json_str = smartsend("/test", data) - >>> # env: MessageEnvelope with all metadata and payloads - >>> # env_json_str: JSON string for publishing - """ - # Generate correlation ID if not provided - cid = correlation_id if correlation_id is not None else str(uuid.uuid4()) - - log_trace(cid, "Starting smartsend for subject: {}".format(subject)) - - # Generate message metadata - msg_id = str(uuid.uuid4()) - - # Process each payload in the list - payloads = [] - - for dataname, payload_data, payload_type in data: - # Serialize data based on type - payload_bytes = _serialize_data(payload_data, payload_type) - - payload_size = len(payload_bytes) - log_trace(cid, "Serialized payload '{}' (payload_type: {}) size: {} bytes".format( - dataname, payload_type, payload_size)) - - # Decision: Direct vs Link - if payload_size < size_threshold: - # Direct path - Base64 encode and send via NATS - # Convert to base64 string for JSON - try: - import ubinascii - payload_b64_str = ubinascii.b2a_base64(payload_bytes).decode('utf-8').strip() - except ImportError: - import base64 - payload_b64_str = base64.b64encode(payload_bytes).decode('utf-8') - - log_trace(cid, "Using direct transport for {} bytes".format(payload_size)) - - # Create MessagePayload for direct transport - payload = MessagePayload( - payload_b64_str, - payload_type, - id=str(uuid.uuid4()), - dataname=dataname, - transport="direct", - encoding="base64", - size=payload_size, - metadata={"payload_bytes": payload_size} - ) - payloads.append(payload) - else: - # Link path - Upload to HTTP server, send URL via NATS - log_trace(cid, "Using link transport, uploading to fileserver") - - # Upload to HTTP server - response = fileserver_upload_handler(fileserver_url, dataname, payload_bytes) - - if response.get("status") != 200: - raise Exception("Failed to upload data to fileserver: {}".format(response.get("status"))) - - url = response.get("url") - log_trace(cid, "Uploaded to URL: {}".format(url)) - - # Create MessagePayload for link transport - payload = MessagePayload( - url, - payload_type, - id=str(uuid.uuid4()), - dataname=dataname, - transport="link", - encoding="none", - size=payload_size, - metadata={} - ) - payloads.append(payload) - - # Create MessageEnvelope with all payloads - env = MessageEnvelope( - subject, - payloads, - correlation_id=cid, - msg_id=msg_id, - msg_purpose=msg_purpose, - sender_name=sender_name, - sender_id=str(uuid.uuid4()), - receiver_name=receiver_name, - receiver_id=receiver_id, - reply_to=reply_to, - reply_to_msg_id=reply_to_msg_id, - broker_url=broker_url, - metadata={} - ) - - msg_json = env.to_json() - - # Publish to NATS if is_publish is True - if is_publish: - nats_conn = NATSConnection(broker_url) - nats_conn.connect() - nats_conn.publish(subject, msg_json) - nats_conn.close() - - # Return tuple of (envelope, json_string) for both direct and link transport - return (env, msg_json) - - -def smartreceive(msg, fileserver_download_handler=_fetch_with_backoff, max_retries=5, - base_delay=100, max_delay=5000): - """Receive and process messages from NATS. - - This function processes incoming NATS messages, handling both direct transport - (base64 decoded payloads) and link transport (URL-based payloads). - - API Standard: - - Returns a dictionary with envelope metadata and 'payloads' field - - payloads field contains list of (dataname, data, payload_type) tuples - - Supports mixed-content messages with different payload types - - Args: - msg: NATS message to process (dict or JSON string with envelope data) - fileserver_download_handler: Function to handle downloading data from file server URLs - Receives: (url, max_retries, base_delay, max_delay, correlation_id) - Returns: bytes (the downloaded data) - max_retries: Maximum retry attempts for fetching URL (default: 5) - base_delay: Initial delay for exponential backoff in ms (default: 100) - max_delay: Maximum delay for exponential backoff in ms (default: 5000) - - Returns: - dict: Envelope dictionary with metadata and 'payloads' field containing list of - (dataname, data, payload_type) tuples - - Envelope fields: correlation_id, msg_id, timestamp, send_to, msg_purpose, - sender_name, sender_id, receiver_name, receiver_id, reply_to, reply_to_msg_id, - broker_url, metadata - - payloads: List of (dataname, data, payload_type) tuples - - Example: - >>> env = smartreceive(msg) - >>> # env contains envelope metadata and payloads field - >>> # env["payloads"] = [(dataname1, data1, payload_type1), ...] - >>> for dataname, data, payload_type in env["payloads"]: - ... print("Received {} of type {}: {}".format(dataname, payload_type, data)) - """ - # Parse the JSON envelope - json_data = msg if isinstance(msg, dict) else json.loads(msg) - correlation_id = json_data.get("correlation_id", "") - log_trace(correlation_id, "Processing received message") - - # Process all payloads in the envelope - payloads_list = [] - - # Get number of payloads - num_payloads = len(json_data.get("payloads", [])) - - for i in range(num_payloads): - payload = json_data["payloads"][i] - transport = payload.get("transport", "") - dataname = payload.get("dataname", "") - - if transport == "direct": - log_trace(correlation_id, - "Direct transport - decoding payload '{}'".format(dataname)) - - # Extract base64 payload from the payload - payload_b64 = payload.get("data", "") - - # Decode Base64 payload - try: - import ubinascii - payload_bytes = ubinascii.a2b_base64(payload_b64.encode('utf-8')) - except ImportError: - import base64 - payload_bytes = base64.b64decode(payload_b64) - - # Deserialize based on type - payload_type = payload.get("payload_type", "") - data = _deserialize_data(payload_bytes, payload_type, correlation_id) - - payloads_list.append((dataname, data, payload_type)) - - elif transport == "link": - # Extract download URL from the payload - url = payload.get("data", "") - log_trace(correlation_id, - "Link transport - fetching '{}' from URL: {}".format(dataname, url)) - - # Fetch with exponential backoff - downloaded_data = fileserver_download_handler( - url, max_retries, base_delay, max_delay, correlation_id - ) - - # Deserialize based on type - payload_type = payload.get("payload_type", "") - data = _deserialize_data(downloaded_data, payload_type, correlation_id) - - payloads_list.append((dataname, data, payload_type)) - - else: - raise ValueError("Unknown transport type for payload '{}': {}".format(dataname, transport)) - - # Replace payloads field with the processed list of (dataname, data, payload_type) tuples - json_data["payloads"] = payloads_list - - return json_data - - -# Utility functions -def generate_uuid(): - """Generate a UUID string.""" - return str(uuid.uuid4()) - - -def get_timestamp(): - """Get current timestamp in ISO format.""" - return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) - - -# Example usage -if __name__ == "__main__": - print("NATSBridge - Bi-Directional Data Bridge") - print("=======================================") - print("This module provides:") - print(" - MessageEnvelope: Message envelope structure with snake_case fields") - print(" - MessagePayload: Payload structure with payload_type field") - print(" - smartsend: Send data via NATS with automatic transport selection") - print(" - smartreceive: Receive and process messages from NATS") - print(" - plik_oneshot_upload: Upload files to HTTP file server") - print(" - _fetch_with_backoff: Fetch data from URLs with retry logic") - print() - print("Usage:") - print(" from nats_bridge import smartsend, smartreceive") - print() - print(" # Send data (list of (dataname, data, payload_type) tuples)") - print(" # Even single payloads must be wrapped in a list") - print(" data = [(\"message\", \"Hello World!\", \"text\")]") - print(" env, env_json_str = smartsend(\"my.subject\", data)") - print() - print(" # On receiver:") - print(" env = smartreceive(msg)") - print(" # env contains envelope metadata and payloads field") - print(" for dataname, data, payload_type in env[\"payloads\"]:") - print(" print(\"Received {} of type {}: {}\".format(dataname, payload_type, data))") - print() - print(" # Mixed-content message example:") - print(" mixed_data = [") - print(" (\"text\", \"Hello!\", \"text\"),") - print(" (\"config\", {\"key\": \"value\"}, \"dictionary\"),") - print(" (\"table\", [{\"id\": 1}], \"table\")") - print(" ]") - print(" smartsend(\"/chat\", mixed_data)") \ No newline at end of file -- 2.49.1 From 78a89523837fac9447af08805b0494bcbfb06533 Mon Sep 17 00:00:00 2001 From: narawat Date: Thu, 26 Feb 2026 16:51:39 +0700 Subject: [PATCH 35/35] update --- src/NATSBridge.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NATSBridge.jl b/src/NATSBridge.jl index c2acaa0..66dd9d9 100644 --- a/src/NATSBridge.jl +++ b/src/NATSBridge.jl @@ -182,7 +182,7 @@ struct msg_envelope_v1 msg_id::String # this message id timestamp::String # message published timestamp (string(Dates.now())) - send_to::String # topic/subject the sender sends to e.g. "/agent/wine/api/v1/prompt" + send_to::String # topic/subject the sender sends this msg to e.g. "/agent/wine/api/v1/prompt" msg_purpose::String # purpose of this message e.g. "ACK", "NACK", "updateStatus", "shutdown", ... sender_name::String # sender name (String) e.g. "agent-wine-web-frontend" sender_id::String # sender id e.g. uuid4() -- 2.49.1