From 396e0848da7e54f78c586585922d4e0186224cdf Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 18 May 2026 19:30:58 +0700 Subject: [PATCH] rename to smartpack n smartunpack --- AI_prompt.md | 33 ++++++---- Cargo.toml | 8 +-- README.md | 84 ++++++++++++------------ docs/architecture.md | 40 +++++------ docs/requirements.md | 20 +++--- docs/specification.md | 74 ++++++++++----------- docs/walkthrough.md | 78 +++++++++++----------- etc.txt | 12 ++-- examples/smartreceive_example.rs | 6 +- examples/smartsend_example.rs | 6 +- src/msghandler-csr.js | 40 +++++------ src/msghandler.jl | 30 ++++----- src/msghandler.js | 40 +++++------ src/msghandler.py | 32 ++++----- src/msghandler.rs | 64 +++++++++--------- src/msghandler_mpy.py | 32 ++++----- test/test_js_mix_payloads_receiver.js | 6 +- test/test_js_mix_payloads_sender.js | 4 +- test/test_julia_mix_payloads_receiver.jl | 10 +-- test/test_julia_mix_payloads_sender.jl | 10 +-- test/test_py_mix_payloads_sender.py | 8 +-- 21 files changed, 323 insertions(+), 314 deletions(-) diff --git a/AI_prompt.md b/AI_prompt.md index fcb77a6..11ffb4a 100644 --- a/AI_prompt.md +++ b/AI_prompt.md @@ -4,7 +4,7 @@ Scenario 2: The "Deep Dive" Analysis (High Bandwidth)Focus: Large Arrow tables, Scenario 3: Live Audio/Signal Processing (Multimedia & Metadata)Focus: Raw binary, bi-directional streaming, headers for metadata.The Action: The JS client captures a 2-second "chunk" of microphone audio. It needs Julia to perform a Fast Fourier Transform (FFT) or AI transcription.The Flow:JS (Sender): Sends the raw binary WAV/PCM data. It uses transport headers to store the metadata ($fs = 44.1kHz$, $channels = 1$) to keep the payload purely binary.Julia (Receiver): Processes the audio and sends back a JSON result (the transcription) and an Arrow Table (the frequency spectrum data).Project Requirement Met: Bi-directional flow involving mixed media (Audio) and technical results (Arrow). Scenario 4: The "Catch-Up" (Persistence & State Sync)Focus: Message persistence, late-joining consumers, state sync.The Action: Julia is constantly publishing "System Health" updates. The JS dashboard is closed for 10 minutes. When the user re-opens the dashboard, they need to see the last 10 minutes of history.The Flow:Transport (Server): Uses a persistence layer with a Limits retention policy.JS (Consumer): Connects and requests a "Replay" from the last 10 minutes. It receives a mix of direct (small updates) and link (historical snapshots) messages.Project Requirement Met: Temporal decoupling—consumers can receive data that was sent while they were offline. -Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service, using a unified message envelope with Claim-Check pattern for large payloads.⚠️ STRICT ARCHITECTURAL CONSTRAINTS (Non-Negotiable)Transport Strategy (Claim-Check Pattern):Direct Path: If payload is < 1MB, send data directly inside the message envelope (Base64 encoded).Link Path: If payload is > 1MB, upload to a shared HTTP fileserver/store. The message must only contain the metadata and the download URL.Tabular Data Format: * MUST use Apache Arrow IPC Stream for all tables/DataFrames. No CSV or standard JSON-serialization of tables allowed.System Symmetry: * Both services must function as Producers AND Consumers.Modular Elegance: * Implementation must be abstracted into a SmartSend function and a SmartReceive handler. The developer calling these functions should not need to care if the data is going via direct or HTTP link.Technical Stack & Use CasesJulia: Arrow.jl, JSON3.jl, HTTP.jl.Node.js: apache-arrow, native fetch.Scenarios to Support: * Large Data: Sending a 500MB Arrow table from Julia $\rightarrow$ JS.Media: Sending a 5MB WAV file from JS $\rightarrow$ Julia.Signals: Sending small JSON control commands ($< 10KB$) directly in the envelope.Implementation Requirements1. Unified JSON Envelope:Define a schema containing: correlation_id (UUID), type (table/binary/json), transport (direct/link), payload (if direct), and url (if link).2. The Julia Module:Implement SmartSend(subject, data, type): Handles Arrow serialization to an IOBuffer, checks size, and manages HTTP uploads for large blobs.Implement SmartReceive(msg): Parses envelope, handles the HTTP fetch with Exponential Backoff (to avoid race conditions), and restores the DataFrame.Include a basic HTTP.listen server to serve as the temporary storage.3. The JavaScript Module:Implement a symmetric SmartSend using native fetch and apache-arrow.Implement a JetStream P... (line truncated to 2000 chars) +Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service, using a unified message envelope with Claim-Check pattern for large payloads.⚠️ STRICT ARCHITECTURAL CONSTRAINTS (Non-Negotiable)Transport Strategy (Claim-Check Pattern):Direct Path: If payload is < 1MB, send data directly inside the message envelope (Base64 encoded).Link Path: If payload is > 1MB, upload to a shared HTTP fileserver/store. The message must only contain the metadata and the download URL.Tabular Data Format: * MUST use Apache Arrow IPC Stream for all tables/DataFrames. No CSV or standard JSON-serialization of tables allowed.System Symmetry: * Both services must function as Producers AND Consumers.Modular Elegance: * Implementation must be abstracted into a smartpack function and a smartunpack handler. The developer calling these functions should not need to care if the data is going via direct or HTTP link.Technical Stack & Use CasesJulia: Arrow.jl, JSON3.jl, HTTP.jl.Node.js: apache-arrow, native fetch.Scenarios to Support: * Large Data: Sending a 500MB Arrow table from Julia $\rightarrow$ JS.Media: Sending a 5MB WAV file from JS $\rightarrow$ Julia.Signals: Sending small JSON control commands ($< 10KB$) directly in the envelope.Implementation Requirements1. Unified JSON Envelope:Define a schema containing: correlation_id (UUID), type (table/binary/json), transport (direct/link), payload (if direct), and url (if link).2. The Julia Module:Implement smartpack(subject, data, type): Handles Arrow serialization to an IOBuffer, checks size, and manages HTTP uploads for large blobs.Implement smartunpack(msg): Parses envelope, handles the HTTP fetch with Exponential Backoff (to avoid race conditions), and restores the DataFrame.Include a basic HTTP.listen server to serve as the temporary storage.3. The JavaScript Module:Implement a symmetric smartpack using native fetch and apache-arrow.Implement a JetStream P... (line truncated to 2000 chars) @@ -25,7 +25,7 @@ Task: Update msghandler.js to reflect recent changes in msghandler.jl and docs Context: msghandler.jl and docs has been updated. Requirements: Source of Truth: Treat the updated msghandler.jl and docs as the definitive source. -API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. +API Consistency: Ensure the Main Package API (e.g., smartpack(), publish_message()) uses consistent naming across all three supported languages. Ecosystem Variance: Low-level native functions (e.g., connect(), JSON.parse()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. @@ -164,7 +164,14 @@ Check the following files: I would like to expand this package (msghandler) to include Rust support. Now help me update Rust implementation of this package at ./src/msghandler.rs. - + +I updated ./src/msghandler-csr.js. Can you check whether files in ./docs needs to be update? +You should check the files sequencially in the following order: +1) ./docs/requirements.md +2) ./docs/specification.md +3) ./docs/architecture.md +4) ./docs/walkthrough.md + @@ -173,7 +180,7 @@ I want to build a client-side-rendering Dioxus-based chat webapp. Dioxus version 0.7+ should be great. I already populate the current folder for the project. my server REST API endpoint is sommpanion.yiem.cc/agent-fronent/api/v1/chat but I didn't run the server yet. A message format is JSON string. -I just placed my custom package for encode and decode message at ./src/msghandler.rs. smartsend() is for encoding and smartreceive() is for decoding. +I just placed my custom package for encode and decode message at ./src/msghandler.rs. smartpack() is for encoding and smartunpack() is for decoding. you may also check the file /home/ton/docker-apps/sommpanion/msghandler/docs/walkthrough.md for more info about my package. You can test whether Dioxus webapp can be build using this command "dx bundle --web --release --debug-symbols=false" @@ -185,8 +192,7 @@ You can test whether Dioxus webapp can be build using this command "dx bundle -- - -I want to build similar webapp. +Do you know about ChatGPT chat interface? I want to build similar webapp. My app should be built as client-side-rendering Dioxus-based (version 0.7+). I already build backend server and I intend to communicate with the webapp using json string that encode the following message envelop: { @@ -233,11 +239,14 @@ I already build backend server and I intend to communicate with the webapp using ] } --- -I already have Rust file named msghandler.rs containing the following functions for the webapp to use: -- smartsend() to encode the above message envelop into json string. -- smartreceive() to decode json string back to message envelop. +I already have this Rust module ./src/msghandler.rs containing the following functions for the webapp to use: +- smartpack() to encode the above message envelop into json string. +- smartunpack() to decode json string back to message envelop. - the msghandler.rs walkthrough is at /home/ton/docker-apps/sommpanion/msghandler/docs/walkthrough.md -The backend server REST API endpoint is "myservice.mydomain.com/subservice/api/v1/chat". I didn't run the server yet. -I already setup the project structure but you can modify the folder as you see fit. Can you implement the app? Use this command "dx bundle --web --release --debug-symbols=false" to check whether the project can be build. -P.S. AI_prompt.md is for me to use. do not read. +MQTT will be used as communication channel between the webapp and the backend. MQTT broker is "mqtt.mydomain.com". I didn't run the broker yet. +I already setup the project structure. Can you implement the app? +To test whether this Dioxus project can be build, you may use this command "dx bundle --web --release --debug-symbols=false" + +P.S. In a Dioxus single-page application (SPA), switching screens can be handled perfectly using standard Rust state matching (often called conditional rendering or state-based routing). + diff --git a/Cargo.toml b/Cargo.toml index b675ca2..580abd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,9 @@ futures = "0.3" tempfile = "3" [[example]] -name = "smartsend_example" -path = "examples/smartsend_example.rs" +name = "smartpack_example" +path = "examples/smartpack_example.rs" [[example]] -name = "smartreceive_example" -path = "examples/smartreceive_example.rs" +name = "smartunpack_example" +path = "examples/smartunpack_example.rs" diff --git a/README.md b/README.md index fc6825b..6590d76 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ msghandler enables seamless communication across multiple platforms through NATS using msghandler data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data; broker_url="nats://localhost:4222") +env, env_json_str = smartpack("/chat/room1", data; broker_url="nats://localhost:4222") println("Message sent!") ``` @@ -120,7 +120,7 @@ println("Message sent!") import msghandler from './src/msghandler_ssr.js'; const data = [["message", "Hello World", "text"]]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { broker_url: "nats://localhost:4222" } @@ -134,7 +134,7 @@ console.log("Message sent!"); import msghandler from './src/msghandler_csr.js'; const data = [["message", "Hello World", "text"]]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { broker_url: "ws://localhost:4222" } @@ -145,10 +145,10 @@ console.log("Message sent!"); #### Python ```python -from msghandler import smartsend +from msghandler import smartpack data = [("message", "Hello World", "text")] -env, env_json_str = await smartsend( +env, env_json_str = await smartpack( "/chat/room1", data, broker_url="nats://localhost:4222" @@ -159,10 +159,10 @@ print("Message sent!") #### MicroPython ```python -from msghandler import smartsend +from msghandler import smartpack data = [("message", "Hello World", "text")] -env, env_json_str = smartsend( +env, env_json_str = smartpack( "/chat/room1", data, broker_url="nats://localhost:4222", @@ -179,12 +179,12 @@ print("Message sent!") All platforms use the same input/output format for payloads: -**Input format for `smartsend`:** +**Input format for `smartpack`:** ``` [(dataname1, data1, type1), (dataname2, data2, type2), ...] ``` -**Output format for `smartreceive`:** +**Output format for `smartunpack`:** ```json { "correlation_id": "...", @@ -204,7 +204,7 @@ All platforms use the same input/output format for payloads: } ``` -### smartsend +### smartpack Sends data either directly via NATS or via a fileserver URL, depending on payload size. @@ -213,7 +213,7 @@ Sends data either directly via NATS or via a fileserver URL, depending on payloa ```julia using msghandler -env, env_json_str = msghandler.smartsend( +env, env_json_str = msghandler.smartpack( subject::String, data::AbstractArray{Tuple{String, Any, String}}; broker_url::String = "nats://localhost:4222", @@ -240,7 +240,7 @@ env, env_json_str = msghandler.smartsend( ```javascript import msghandler from './src/msghandler_ssr.js'; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( subject, data, // Array of [dataname, data, type] tuples { @@ -269,7 +269,7 @@ const [env, env_json_str] = await msghandler.smartsend( ```javascript import msghandler from './src/msghandler_csr.js'; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( subject, data, { @@ -298,7 +298,7 @@ const [env, env_json_str] = await msghandler.smartsend( ```python from msghandler import msghandler -env, env_json_str = await msghandler.smartsend( +env, env_json_str = await msghandler.smartpack( subject: str, data: List[Tuple[str, Any, str]], broker_url: str = "nats://localhost:4222", @@ -326,7 +326,7 @@ env, env_json_str = await msghandler.smartsend( from msghandler import msghandler # Limited to direct transport (< 100KB threshold) -env, env_json_str = msghandler.smartsend( +env, env_json_str = msghandler.smartpack( subject, data, # List of (dataname, data, type) tuples broker_url="nats://localhost:4222", @@ -335,7 +335,7 @@ env, env_json_str = msghandler.smartsend( # Returns: Tuple[Dict, str] ``` -### smartreceive +### smartunpack Receives and processes messages from NATS, handling both direct and link transport. @@ -344,7 +344,7 @@ Receives and processes messages from NATS, handling both direct and link transpo ```julia using msghandler -env = msghandler.smartreceive( +env = msghandler.smartunpack( msg::NATS.Msg; fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -359,7 +359,7 @@ env = msghandler.smartreceive( ```javascript import msghandler from './src/msghandler_ssr.js'; -const env = await msghandler.smartreceive( +const env = await msghandler.smartunpack( msg, { fileserver_download_handler: msghandler.fetchWithBackoff, @@ -376,7 +376,7 @@ const env = await msghandler.smartreceive( ```javascript import msghandler from './src/msghandler_csr.js'; -const env = await msghandler.smartreceive( +const env = await msghandler.smartunpack( msg, { fileserver_download_handler: msghandler.fetchWithBackoff, @@ -393,7 +393,7 @@ const env = await msghandler.smartreceive( ```python from msghandler import msghandler -env = await msghandler.smartreceive( +env = await msghandler.smartunpack( msg, fileserver_download_handler=fetch_with_backoff, max_retries=5, @@ -408,7 +408,7 @@ env = await msghandler.smartreceive( ```python from msghandler import msghandler -env = msghandler.smartreceive( +env = msghandler.smartunpack( msg, fileserver_download_handler=_sync_fileserver_download, max_retries=3, @@ -452,7 +452,7 @@ data = [ ("large_document", large_file_data, "binary") ] -env, env_json_str = smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") +env, env_json_str = smartpack("/chat/room1", data; fileserver_url="http://localhost:8080") ``` #### JavaScript (Node.js) @@ -466,7 +466,7 @@ const data = [ ["large_document", largeFileData, "binary"] ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { fileserver_url: 'http://localhost:8080' } @@ -484,7 +484,7 @@ const data = [ ["large_document", largeFileData, "binary"] ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { broker_url: 'ws://localhost:4222', fileserver_url: 'http://localhost:8080' } @@ -502,7 +502,7 @@ data = [ ("large_document", large_file_data, "binary") ] -env, env_json_str = await msghandler.smartsend( +env, env_json_str = await msghandler.smartpack( "/chat/room1", data, fileserver_url="http://localhost:8080" @@ -525,7 +525,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data) +env, env_json_str = smartpack("/device/config", data) ``` #### JavaScript (Node.js) @@ -539,7 +539,7 @@ const config = { update_interval: 60 }; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/device/config", [["config", config, "dictionary"]] ); @@ -557,7 +557,7 @@ config = { } data = [("config", config, "dictionary")] -env, env_json_str = await msghandler.smartsend("/device/config", data) +env, env_json_str = await msghandler.smartpack("/device/config", data) ``` ### Example 3: Table Data (Arrow IPC) @@ -577,7 +577,7 @@ df = DataFrame( ) data = [("students", df, "arrowtable")] -env, env_json_str = smartsend("/data/analysis", data) +env, env_json_str = smartpack("/data/analysis", data) ``` #### JavaScript (Node.js) @@ -591,7 +591,7 @@ const df = [ { id: 3, name: "Charlie", score: 92 } ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/data/analysis", [["students", df, "arrowtable"]] ); @@ -610,7 +610,7 @@ df = pd.DataFrame({ }) data = [("students", df, "arrowtable")] -env, env_json_str = await msghandler.smartsend("/data/analysis", data) +env, env_json_str = await msghandler.smartpack("/data/analysis", data) ``` #### JavaScript (Browser) @@ -626,7 +626,7 @@ const df = [ { id: 3, name: "Charlie", score: 92 } ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/data/analysis", [["students", df, "jsontable"]], // Use jsontable for browser { broker_url: 'ws://localhost:4222' } @@ -643,7 +643,7 @@ Bi-directional communication with reply-to support. using msghandler # Requester -env, env_json_str = smartsend( +env, env_json_str = smartpack( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; broker_url="nats://localhost:4222", @@ -652,9 +652,9 @@ env, env_json_str = smartsend( # Receiver (in separate application) msg = NATS.subscription.next() -env = smartreceive(msg) +env = smartunpack(msg) # Process request and send response -response_env, response_json = smartsend( +response_env, response_json = smartpack( "/device/response", [("result", Dict("value" => 42), "dictionary")], reply_to="/device/command", @@ -668,7 +668,7 @@ response_env, response_json = smartsend( import msghandler from './src/msghandler_ssr.js'; // Requester -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/device/command", [["command", { action: "read_sensor" }, "dictionary"]], { broker_url: 'nats://localhost:4222', reply_to: '/device/response' } @@ -676,9 +676,9 @@ const [env, env_json_str] = await msghandler.smartsend( // Receiver (in separate application) // const msg = await natsConsumer.next(); -// const env = await msghandler.smartreceive(msg); +// const env = await msghandler.smartunpack(msg); // Process request and send response -// const response_env, response_json = await msghandler.smartsend( +// const response_env, response_json = await msghandler.smartpack( // "/device/response", // [["result", { value: 42 }, "dictionary"]], // { reply_to: '/device/command', reply_to_msg_id: env.msg_id } @@ -691,7 +691,7 @@ const [env, env_json_str] = await msghandler.smartsend( from msghandler import msghandler # Requester -env, env_json_str = await msghandler.smartsend( +env, env_json_str = await msghandler.smartpack( "/device/command", [("command", {"action": "read_sensor"}, "dictionary")], broker_url="nats://localhost:4222", @@ -700,9 +700,9 @@ env, env_json_str = await msghandler.smartsend( # Receiver (in separate application) # msg = await nats_consumer.next() -# env = await msghandler.smartreceive(msg) +# env = await msghandler.smartunpack(msg) # Process request and send response -# response_env, response_json = await msghandler.smartsend( +# response_env, response_json = await msghandler.smartpack( # "/device/response", # [("result", {"value": 42}, "dictionary")], # reply_to="/device/command", @@ -895,7 +895,7 @@ node build.js import msghandlerCSR from './dist/msghandler-csr-bundle.js'; // Use the library - const [env, envJson] = await msghandlerCSR.smartsend( + const [env, envJson] = await msghandlerCSR.smartpack( "/chat/user/v1/message", [["msg", "Hello", "text"]], { broker_url: "wss://nats.example.com" } diff --git a/docs/architecture.md b/docs/architecture.md index 458677f..1259106 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -130,8 +130,8 @@ flowchart TD ```mermaid flowchart TD subgraph "msghandler Module" - SmartSend[smartsend Function] - SmartReceive[smartreceive Function] + smartpack[smartpack Function] + smartunpack[smartunpack Function] Serialize[_serialize_data] Deserialize[_deserialize_data] @@ -149,18 +149,18 @@ flowchart TD Envelope[msg_envelope_v1 Struct] end - SmartSend --> Serialize - SmartSend --> EnvelopeToJson - SmartSend --> FileServerUpload + smartpack --> Serialize + smartpack --> EnvelopeToJson + smartpack --> FileServerUpload - SmartReceive --> Deserialize - SmartReceive --> FileServerDownload + smartunpack --> Deserialize + smartunpack --> FileServerDownload EnvelopeToJson --> Envelope Serialize --> Payload - style SmartSend fill:#d1fae5,stroke:#10b981 - style SmartReceive fill:#d1fae5,stroke:#10b981 + style smartpack fill:#d1fae5,stroke:#10b981 + style smartunpack fill:#d1fae5,stroke:#10b981 style FileServerUpload fill:#fef3c7,stroke:#f59e0b style FileServerDownload fill:#fef3c7,stroke:#f59e0b ``` @@ -173,8 +173,8 @@ flowchart TD | Component | Purpose | Platform Support | |-----------|---------|------------------| -| **smartsend** | Send data with automatic transport selection, returns (envelope, json_string) for caller to publish via transport | All | -| **smartreceive** | Receive and process messages from JSON string | All | +| **smartpack** | Send data with automatic transport selection, returns (envelope, json_string) for caller to publish via transport | All | +| **smartunpack** | Receive and process messages from JSON string | All | | **_serialize_data** | Serialize data according to payload type | All | | **_deserialize_data** | Deserialize bytes to native data types | All | | **envelope_to_json** | Convert msg_envelope_v1 struct to JSON string | All | @@ -187,7 +187,7 @@ flowchart TD ```mermaid flowchart TD - A[User calls smartsend subject data] --> B[Process each payload] + A[User calls smartpack subject data] --> B[Process each payload] B --> C{Calculate serialized size} C -->|Size < Threshold| D[Direct Transport] C -->|Size >= Threshold| E[Link Transport] @@ -349,7 +349,7 @@ flowchart TD ```mermaid flowchart TD - A[smartsend called] --> B[Serialize payload] + A[smartpack called] --> B[Serialize payload] B --> C[Calculate size] C --> D{Size < Threshold?} @@ -560,7 +560,7 @@ pub enum Payload { } // Configuration via builder pattern -pub struct SmartsendOptions { +pub struct smartpackOptions { pub broker_url: String, pub fileserver_url: String, pub fileserver_upload_handler: Option>, @@ -577,7 +577,7 @@ let conn = transport_client::connect(DEFAULT_BROKER_URL).await?; // Subscribe and process messages let mut sub = conn.subscribe("/agent/wine/api/v1/analyze")?; for msg in sub.messages() { - let envelope = smartreceive(&String::from_utf8_lossy(&msg.payload), &Default::default()).await?; + let envelope = smartunpack(&String::from_utf8_lossy(&msg.payload), &Default::default()).await?; // Access deserialized payloads by type for payload in &envelope.payloads { match payload.payload_type.as_str() { @@ -840,11 +840,11 @@ flowchart TD | - | - | Updated diagrams to use generic "Message Broker" instead of "NATS Server" | All sections | | - | - | Updated code examples to use transport-agnostic patterns | All sections | | - | - | Removed NATS client packages from external dependencies | All sections | -| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartreceive` deserialization changes | All sections | -| - | - | `smartreceive` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | +| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | +| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | | - | - | Added `plik_upload_file` convenience function to component table | specification.md:13 | | - | - | Fixed Rust payload access pattern (data is String, not Payload enum) | All sections | -| - | - | Fixed `SmartsendOptions.fileserver_upload_handler` type to `Arc` | specification.md:13 | +| - | - | Fixed `smartpackOptions.fileserver_upload_handler` type to `Arc` | specification.md:13 | | - | - | Removed `metadata` from link transport examples (now `None`/omitted) | specification.md:3 | | - | - | Removed duplicate footer text | All sections | | 2026-05-13 | 1.3.0 | Added Rust support with tokio, serde, and arrow2 | All sections | @@ -854,9 +854,9 @@ flowchart TD | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | | - | - | Removed publish_message component (commented out in source) | | - | - | Removed NATSClient and NATSConnectionPool classes (not in ground truth) | -| - | - | Updated smartsend to return JSON for caller to publish via transport | +| - | - | Updated smartpack to return JSON for caller to publish via transport | | - | - | Updated component diagram to match actual module structure | -| - | - | Updated data flow to show smartsend returns JSON for caller to publish | +| - | - | Updated data flow to show smartpack returns JSON for caller to publish | | - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | | 2026-03-15 | 1.1.0 | JavaScript connection management | | - | - | Added NATSClient with keepAlive support | diff --git a/docs/requirements.md b/docs/requirements.md index fc20b68..ce93f63 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -54,7 +54,7 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless |---------|-------------| | Cross-platform interoperability | Seamless data exchange between Julia, JavaScript, Python, Dart, Rust, and MicroPython | | Intelligent transport selection | Direct transport (<0.5MB) vs Link transport (≥0.5MB) based on payload size | -| Unified API | Consistent `smartsend()` and `smartreceive()` functions across all platforms | +| Unified API | Consistent `smartpack()` and `smartunpack()` functions across all platforms | | Multi-payload support | List of (dataname, data, type) tuples with appropriate handling | | File server integration | Plik one-shot upload and custom HTTP server support | | Reliability features | Exponential backoff retry and correlation ID propagation | @@ -323,10 +323,10 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless ## 11. API Contract -### 11.1 smartsend Signature +### 11.1 smartpack Signature ```julia -function smartsend( +function smartpack( subject::String, data::AbstractArray{Tuple{String, T1, String}, 1}; broker_url::String = DEFAULT_BROKER_URL, @@ -345,12 +345,12 @@ function smartsend( )::Tuple{msg_envelope_v1, String} where {T1<:Any} ``` -**Note**: Publishing via the transport layer is the caller's responsibility. `smartsend` returns `(env::msg_envelope_v1, env_json_str::String)`. +**Note**: Publishing via the transport layer is the caller's responsibility. `smartpack` returns `(env::msg_envelope_v1, env_json_str::String)`. -### 11.2 smartreceive Signature +### 11.2 smartunpack Signature ```julia -function smartreceive( +function smartunpack( msg_json_str::String; fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -359,9 +359,9 @@ function smartreceive( )::JSON.Object{String, Any} ``` -**Note**: Pass the payload string from the transport subscription to `smartreceive`. The input is the JSON string payload from the transport message, not the transport message object directly. +**Note**: Pass the payload string from the transport subscription to `smartunpack`. The input is the JSON string payload from the transport message, not the transport message object directly. -**Note**: Pass the payload from the transport subscription to `smartreceive`. +**Note**: Pass the payload from the transport subscription to `smartunpack`. --- @@ -411,8 +411,8 @@ function smartreceive( | - | - | Updated all NATS references to generic "transport layer"/"message broker" | | - | - | Removed NATS client packages from dependencies tables | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | -| - | - | Fixed smartsend signature: removed is_publish, NATS_connection; added sender_name | -| - | - | Fixed smartreceive signature: takes msg_json_str::String instead of msg::NATS.Msg | +| - | - | Fixed smartpack signature: removed is_publish, NATS_connection; added sender_name | +| - | - | Fixed smartunpack signature: takes msg_json_str::String instead of msg::NATS.Msg | | - | - | Fixed size_threshold default from 1,000,000 to 500,000 | | - | - | Updated FR-013/FR-014 to reflect caller responsibility for NATS publishing | | - | - | Updated FR-008/FR-009 to include file path upload overload | diff --git a/docs/specification.md b/docs/specification.md index 2b32c91..df773f6 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -13,8 +13,8 @@ This document defines the **technical contract** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. This specification serves as the single source of truth for: -- **Inputs**: What data structures are accepted by `smartsend()` -- **Outputs**: What data structures are returned by `smartreceive()` +- **Inputs**: What data structures are accepted by `smartpack()` +- **Outputs**: What data structures are returned by `smartunpack()` - **Data Shapes**: Exact field names, types, and constraints - **Error Codes**: Standardized error responses for failure scenarios @@ -24,7 +24,7 @@ This specification serves as the single source of truth for: |----------------------|-------------------|-------------| | Section 2 (Message Envelope) | FR-012, FR-013, NFR-101, NFR-102 | Message envelope structure and validation | | Section 3 (Payload Schema) | FR-001, FR-002, FR-003, FR-004, NFR-101, NFR-102 | Payload structure and field definitions | -| Section 4 (Payload Format) | FR-006, FR-007 | Tuple format for smartsend() | +| Section 4 (Payload Format) | FR-006, FR-007 | Tuple format for smartpack() | | Section 5 (Enumerations) | FR-003, FR-004, FR-006, NFR-101 | Enumerations for transport and encoding | | Section 6 (Transport Protocols) | FR-003, FR-004, NFR-104, NFR-105 | Direct and link transport protocols | | Section 7 (Size Thresholds) | FR-004, FR-005, NFR-104, NFR-105 | Size thresholds for transport selection | @@ -143,9 +143,9 @@ This specification serves as the single source of truth for: ## Payload Format -### Tuple Format for `smartsend()` +### Tuple Format for `smartpack()` -The `smartsend()` function accepts data as an array of tuples with the format: +The `smartpack()` function accepts data as an array of tuples with the format: ``` ("data_name", data, "data_type") @@ -161,17 +161,17 @@ The `smartsend()` function accepts data as an array of tuples with the format: ```julia # Julia -smartsend("/chat/user/v1/message", [("msg", "Hello World", "text")]) +smartpack("/chat/user/v1/message", [("msg", "Hello World", "text")]) ``` ```python # Python -await smartsend("/chat/user/v1/message", [("msg", "Hello World", "text")]) +await smartpack("/chat/user/v1/message", [("msg", "Hello World", "text")]) ``` ```typescript // JavaScript -await smartsend("/chat/user/v1/message", [["msg", "Hello World", "text"]]); +await smartpack("/chat/user/v1/message", [["msg", "Hello World", "text"]]); ``` ### Multiple Payloads Example @@ -182,7 +182,7 @@ data = [ ("msg", "Hello", "text"), ("img", binary_data, "image") ] -smartsend("/agent/v1/process", data) +smartpack("/agent/v1/process", data) ``` ```python @@ -191,7 +191,7 @@ data = [ ("msg", "Hello", "text"), ("img", binary_data, "image") ] -await smartsend("/agent/v1/process", data) +await smartpack("/agent/v1/process", data) ``` ### Data Type Mapping @@ -411,12 +411,12 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa ## API Contract -### `smartsend` Function Signature +### `smartpack` Function Signature #### Julia ```julia -function smartsend( +function smartpack( subject::String, data::AbstractArray{Tuple{String, T1, String}, 1}; broker_url::String = DEFAULT_BROKER_URL, @@ -440,7 +440,7 @@ function smartsend( #### Python ```python -async def smartsend( +async def smartpack( subject: str, data: List[Tuple[str, Any, str]], broker_url: str = DEFAULT_BROKER_URL, @@ -464,7 +464,7 @@ async def smartsend( #### JavaScript (Node.js) ```typescript -async function smartsend( +async function smartpack( subject: string, data: Array<[string, any, string]>, options?: { @@ -490,7 +490,7 @@ async function smartsend( #### JavaScript (Browser) ```typescript -async function smartsend( +async function smartpack( subject: string, data: Array<[string, any, string]>, options?: { @@ -516,7 +516,7 @@ async function smartsend( #### MicroPython ```python -def smartsend( +def smartpack( subject: str, data: List[Tuple[str, Any, str]], size_threshold: int = 100_000, # Lower threshold for memory constraints @@ -529,7 +529,7 @@ def smartsend( #### Dart (Desktop/Flutter) ```dart -Future<[Map, String]> smartsend( +Future<[Map, String]> smartpack( String subject, List> data, { String brokerUrl = DEFAULT_BROKER_URL, @@ -553,7 +553,7 @@ Future<[Map, String]> smartsend( #### Dart Web ```dart -Future<[Map, String]> smartsend( +Future<[Map, String]> smartpack( String subject, List> data, { String brokerUrl = DEFAULT_BROKER_URL, @@ -578,14 +578,14 @@ Future<[Map, String]> smartsend( #### Rust ```rust -pub async fn smartsend( +pub async fn smartpack( subject: &str, data: &[(String, Payload, String)], - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), msghandlerError> -// SmartsendOptions struct -pub struct SmartsendOptions { +// smartpackOptions struct +pub struct smartpackOptions { pub broker_url: String, pub fileserver_url: String, pub fileserver_upload_handler: Option, @@ -636,12 +636,12 @@ pub struct MsgEnvelopeV1 { **Note**: Publishing via the transport layer is the caller's responsibility. Returns `Result<(MsgEnvelopeV1, String), msghandlerError>`. Uses `serde` for JSON serialization. -### `smartreceive` Function Signature +### `smartunpack` Function Signature #### Julia ```julia -function smartreceive( +function smartunpack( msg_json_str::String; # Pass payload from transport subscription fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -655,7 +655,7 @@ function smartreceive( #### Python ```python -async def smartreceive( +async def smartunpack( msg_json_str: str, # JSON string from transport message payload fileserver_download_handler: Callable = fetch_with_backoff, max_retries: int = 5, @@ -669,7 +669,7 @@ async def smartreceive( #### JavaScript (Node.js) ```typescript -async function smartreceive( +async function smartunpack( msg_json_str: string, // JSON string from transport message payload options?: { fileserver_download_handler?: Function; @@ -683,7 +683,7 @@ async function smartreceive( #### JavaScript (Browser) ```typescript -async function smartreceive( +async function smartunpack( msg_json_str: string, // JSON string from transport message payload options?: { fileserver_download_handler?: Function; @@ -699,7 +699,7 @@ async function smartreceive( #### MicroPython ```python -def smartreceive(msg_json_str: str, **kwargs) -> Dict[str, Any]: +def smartunpack(msg_json_str: str, **kwargs) -> Dict[str, Any]: ``` **Note**: Input is the JSON string payload from the transport message. @@ -707,7 +707,7 @@ def smartreceive(msg_json_str: str, **kwargs) -> Dict[str, Any]: #### Dart (Desktop/Flutter) ```dart -Future> smartreceive( +Future> smartunpack( Map msg_json_str, // JSON object from transport message payload { Function? fileserverDownloadHandler, @@ -722,7 +722,7 @@ Future> smartreceive( #### Dart Web ```dart -Future> smartreceive( +Future> smartunpack( Map msg_json_str, // JSON object from transport message payload { Function? fileserverDownloadHandler, @@ -737,13 +737,13 @@ Future> smartreceive( #### Rust ```rust -pub async fn smartreceive( +pub async fn smartunpack( msg_json_str: &str, // JSON string from transport message payload - options: &SmartreceiveOptions, + options: &smartunpackOptions, ) -> Result -// SmartreceiveOptions struct -pub struct SmartreceiveOptions { +// smartunpackOptions struct +pub struct smartunpackOptions { pub fileserver_download_handler: Option, pub max_retries: u32, pub base_delay: u64, @@ -933,7 +933,7 @@ The browser implementation ([`src/msghandler_csr.js`](../src/msghandler_csr.js)) ```mermaid flowchart TD - A[User calls smartsend subject data] --> B[Serialize payload according to payload_type] + A[User calls smartpack subject data] --> B[Serialize payload according to payload_type] B --> C{Calculate serialized size} C -->|Size < Threshold| D[Direct Transport: Encode as Base64] C -->|Size >= Threshold| E[Link Transport: Upload to file server] @@ -1156,8 +1156,8 @@ flowchart TD | - | - | Updated all NATS references to generic "transport layer"/"message broker" | All | | - | - | Removed NATS client packages from dependencies tables | All | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All | -| - | - | Updated smartsend signatures: removed is_publish, nats_connection; added sender_name | FR-001 through FR-014 | -| - | - | Updated smartreceive signatures: takes msg_json_str::String instead of msg | FR-001 through FR-014 | +| - | - | Updated smartpack signatures: removed is_publish, nats_connection; added sender_name | FR-001 through FR-014 | +| - | - | Updated smartunpack signatures: takes msg_json_str::String instead of msg | FR-001 through FR-014 | | - | - | Removed publishMessage function and NATSClient/NATSConnectionPool classes from browser section | FR-013, FR-014 | | - | - | Added plik_oneshot_upload(filepath) overload to file server interface | FR-008, FR-009 | | - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | FR-003, FR-004 | diff --git a/docs/walkthrough.md b/docs/walkthrough.md index a489997..9ee50c2 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -44,7 +44,7 @@ flowchart TB subgraph msghandler["msghandler Module"] direction TB - subgraph Sender["Sender (smartsend)"] + subgraph Sender["Sender (smartpack)"] direction LR S1["Data Tuples
[(dataname, data, type)]"] S2["Serialize Data"] @@ -60,7 +60,7 @@ flowchart TB S5 --> S6 end - subgraph Receiver["Receiver (smartreceive)"] + subgraph Receiver["Receiver (smartunpack)"] direction LR R1["Subscribe via transport"] R2["Parse Envelope"] @@ -101,7 +101,7 @@ flowchart TB |-----------|-------------|-----------| | **Claim-Check Pattern** | Large payloads uploaded to HTTP server, URL sent via transport | Transport has message size limits; avoids overflow | | **Automatic Transport Selection** | Direct (< threshold) vs Link (≥ threshold) based on size | Optimizes memory vs network I/O trade-off | -| **Cross-Platform API** | Consistent `smartsend()`/`smartreceive()` across all platforms | Simplifies developer experience | +| **Cross-Platform API** | Consistent `smartpack()`/`smartunpack()` across all platforms | Simplifies developer experience | | **Exponential Backoff** | Retry downloads with increasing delays | Handles transient failures gracefully | --- @@ -118,7 +118,7 @@ A JavaScript chat webapp wants to send mixed payloads (text message + user avata ```javascript // JavaScript (Browser or Node.js) -const [env, msgJson] = await msghandler.smartsend( +const [env, msgJson] = await msghandler.smartpack( "/agent/wine/api/v1/prompt", [ ["msg", "Hello! I'm Ton.", "text"], @@ -225,14 +225,14 @@ msghandler builds the message envelope: **Rationale**: - The transport layer provides message delivery (NATS, MQTT, WebSocket, etc.) - JSON format ensures cross-platform compatibility -- `smartsend()` returns `(env, msgJson)` - caller handles publishing via their chosen transport +- `smartpack()` returns `(env, msgJson)` - caller handles publishing via their chosen transport #### Step 6: Julia Backend Receives Message ```julia # Julia backend transport_msg = transport_subscription.next() # Get message from transport -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # env["payloads"] is now: # [ @@ -242,7 +242,7 @@ env = smartreceive(String(transport_msg.payload)) ``` **Rationale**: -- `smartreceive()` handles both transport types automatically +- `smartunpack()` handles both transport types automatically - Deserialization is type-aware based on `payload_type` - Returns consistent tuple format regardless of transport @@ -253,7 +253,7 @@ env = smartreceive(String(transport_msg.payload)) response_text = "Hello Ton! I'm the AI assistant." generated_image = generate_ai_image(response_text) -env, msg_json = smartsend( +env, msg_json = smartpack( "/agent/wine/api/v1/response", [ ("response", response_text, "text"), @@ -282,7 +282,7 @@ A JavaScript webapp wants to upload a large file (10MB) to a Julia backend for p #### Step 1: JavaScript Webapp Sends Large File ```javascript -const [env, msgJson] = await msghandler.smartsend( +const [env, msgJson] = await msghandler.smartpack( "/agent/wine/api/v1/process", [ ["file", largeFileData, "binary"] @@ -358,7 +358,7 @@ const response = await plikOneshotUpload( ```julia # Julia backend transport_msg = transport_subscription.next() -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # msghandler automatically: # 1. Extracts URL from payload @@ -386,7 +386,7 @@ A Python application sends tabular data (pandas DataFrame) to a Julia backend fo ```python # Python import pandas as pd -from msghandler import smartsend +from msghandler import smartpack df = pd.DataFrame({ "id": [1, 2, 3], @@ -394,7 +394,7 @@ df = pd.DataFrame({ "score": [95, 88, 92] }) -env, msg_json = await smartsend( +env, msg_json = await smartpack( "/agent/wine/api/v1/analyze", [("data", df, "arrowtable")], broker_url=DEFAULT_BROKER_URL, @@ -431,7 +431,7 @@ arrow_bytes = buf.getvalue() ```julia # Julia backend transport_msg = transport_subscription.next() -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # env["payloads"][1] is now: # ("data", DataFrame with id, name, score columns, "arrowtable") @@ -449,7 +449,7 @@ env = smartreceive(String(transport_msg.payload)) results = analyze_data(env["payloads"][1][2]) # Send results back -env, msg_json = smartsend( +env, msg_json = smartpack( "/agent/wine/api/v1/results", [("results", results, "arrowtable")], reply_to = "/python/worker/v1/results" @@ -475,7 +475,7 @@ A Rust service needs to process messages from a Julia analytics pipeline and sen ```rust // Rust service - using tokio async runtime -use msghandler::{smartreceive, MsgEnvelopeV1}; +use msghandler::{smartunpack, MsgEnvelopeV1}; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; #[tokio::main] @@ -486,7 +486,7 @@ async fn main() { let mut sub = conn.subscribe("/agent/wine/api/v1/analyze").unwrap(); for msg in sub.messages() { - let envelope = smartreceive( + let envelope = smartunpack( &String::from_utf8_lossy(&msg.payload), &Default::default(), ).await.unwrap(); @@ -495,7 +495,7 @@ async fn main() { for payload in &envelope.payloads { match payload.payload_type.as_str() { "arrowtable" => { - // Data is base64-encoded Arrow IPC bytes after smartreceive() + // Data is base64-encoded Arrow IPC bytes after smartunpack() let arrow_bytes = BASE64.decode(&payload.data).unwrap(); println!("Received arrowtable payload ({} bytes)", arrow_bytes.len()); }, @@ -522,19 +522,19 @@ async fn main() { **Rationale**: - **serde serialization**: Automatic JSON deserialization to `MsgEnvelopeV1` - **tokio runtime**: Efficient async I/O for transport and HTTP operations -- **smartreceive deserialization**: Payload data is deserialized and stored as strings in `payload.data` +- **smartunpack deserialization**: Payload data is deserialized and stored as strings in `payload.data` - **Type dispatch**: `payload_type` field determines how to interpret the `data` string #### Step 2: Rust Service Sends Processed Results ```rust // Rust service sends results back with mixed payload types -use msghandler::{smartsend, Payload, SmartsendOptions}; +use msghandler::{smartpack, Payload, smartpackOptions}; let results_df = /* processed Arrow table */; let result_bytes = /* serialize to Arrow IPC */; -let (envelope, json_str) = smartsend( +let (envelope, json_str) = smartpack( "/agent/wine/api/v1/results", &[ ( @@ -548,7 +548,7 @@ let (envelope, json_str) = smartsend( "text".to_string(), ), ], - &SmartsendOptions { + &smartpackOptions { broker_url: DEFAULT_BROKER_URL.to_string(), reply_to: "/python/worker/v1/results".to_string(), msg_purpose: "chat".to_string(), @@ -561,7 +561,7 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; ``` **Rationale**: -- **Builder pattern**: `SmartsendOptions` provides clean configuration +- **Builder pattern**: `smartpackOptions` provides clean configuration - **Enum-based payloads**: Type safety prevents sending incorrect data types - **Default options**: sensible defaults reduce boilerplate - **Result**: idiomatic Rust error handling @@ -570,7 +570,7 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; ```python # Python backend receives Rust response -env = await smartreceive(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)) # env["payloads"][0] is now: # ("results", arrow_table_data, "arrowtable") @@ -589,7 +589,7 @@ env = await smartreceive(str(transport_msg.payload)) // Rust service sends large binary file via link transport let large_file_data: Vec = std::fs::read("/data/large_dataset.parquet")?; -let (envelope, json_str) = smartsend( +let (envelope, json_str) = smartpack( "/agent/wine/api/v1/upload", &[ ( @@ -598,7 +598,7 @@ let (envelope, json_str) = smartsend( "binary".to_string(), ), ], - &SmartsendOptions { + &smartpackOptions { broker_url: DEFAULT_BROKER_URL.to_string(), fileserver_url: DEFAULT_FILESERVER_URL.to_string(), size_threshold: DEFAULT_SIZE_THRESHOLD, // threshold triggers link transport @@ -627,7 +627,7 @@ A MicroPython sensor device sends sensor readings to a Python backend. ```python # MicroPython -from msghandler import smartsend +from msghandler import smartpack sensor_data = { "temperature": 25.5, @@ -635,7 +635,7 @@ sensor_data = { "pressure": 1013.25 } -env, msg_json = smartsend( +env, msg_json = smartpack( "/sensor/device/v1/readings", [("data", sensor_data, "dictionary")], broker_url=DEFAULT_BROKER_URL, @@ -667,7 +667,7 @@ payload_b64 = base64.b64encode(json_bytes).decode('ascii') ```python # Python backend transport_msg = await transport_consumer.next() -env = await smartreceive(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)) # env["payloads"][0] is now: # ("data", {"temperature": 25.5, "humidity": 60.0, ...}, "dictionary") @@ -692,7 +692,7 @@ Multiple platforms (JavaScript, Python, Julia) communicate in a chat application ```javascript // JavaScript (Frontend) -const [env, msgJson] = await msghandler.smartsend( +const [env, msgJson] = await msghandler.smartpack( "/chat/user/v1/message", [ ["text", "Check this out!", "text"], @@ -716,7 +716,7 @@ const [env, msgJson] = await msghandler.smartsend( ```python # Python (Backend) transport_msg = await transport_consumer.next() -env = await smartreceive(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)) # env["payloads"] is now: # [ @@ -735,7 +735,7 @@ env = await smartreceive(str(transport_msg.payload)) ```julia # Julia (Backend) transport_msg = transport_subscription.next() -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # env["payloads"] is now: # [ @@ -755,7 +755,7 @@ Each platform can reply using the same API: ```python # Python reply -await smartsend( +await smartpack( "/chat/user/v1/reply", [("response", "Nice!", "text")], reply_to="/chat/user/v1/message" @@ -764,7 +764,7 @@ await smartsend( ```julia # Julia reply -smartsend( +smartpack( "/chat/user/v1/reply", [("response", "Nice!", "text")], reply_to="/chat/user/v1/message" @@ -773,7 +773,7 @@ smartsend( ```javascript // JavaScript reply -await msghandler.smartsend( +await msghandler.smartpack( "/chat/user/v1/reply", [["response", "Nice!", "text"]], { reply_to: "/chat/user/v1/message" } @@ -827,14 +827,14 @@ Every message includes a `correlation_id`: correlation_id = string(uuid4()) # Use throughout the flow -log_trace(correlation_id, "Starting smartsend") +log_trace(correlation_id, "Starting smartpack") log_trace(correlation_id, "Serialized payload size: 100 bytes") log_trace(correlation_id, "Published to transport") ``` **Log Format**: ``` -[2026-03-13T16:30:00.000Z] [Correlation: abc123...] Starting smartsend +[2026-03-13T16:30:00.000Z] [Correlation: abc123...] Starting smartpack [2026-03-13T16:30:00.001Z] [Correlation: abc123...] Serialized payload size: 100 bytes [2026-03-13T16:30:00.002Z] [Correlation: abc123...] Published to transport ``` @@ -928,8 +928,8 @@ log_trace(correlation_id, "Published to transport") | - | - | Removed all NATS-specific references from walkthrough | All sections | | - | - | Updated code examples to use transport-agnostic patterns | All sections | | - | - | Updated diagrams to remove NATS-specific labels | All sections | -| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartreceive` deserialization changes | All sections | -| - | - | `smartreceive` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | +| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | +| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | | - | - | Added `plik_upload_file` convenience function documentation | specification.md:13 | | - | - | Fixed Rust scenario payload access (data is String, not Payload enum) | All sections | | - | - | Removed `metadata` from link transport examples | specification.md:3 | @@ -937,7 +937,7 @@ log_trace(correlation_id, "Published to transport") | - | - | Added Rust user scenario (User Scenario 4) | specification.md:11 (Rust API) | | - | - | Updated scenario numbering (MicroPython → Scenario 5, Cross-Platform → Scenario 6) | All sections | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All sections | -| - | - | Updated smartreceive calls to use transport payload pattern | All sections | +| - | - | Updated smartunpack calls to use transport payload pattern | All sections | | - | - | Removed NATSClient.publish() calls (caller responsible for transport publishing) | All sections | | - | - | Removed is_publish and nats_connection parameter references | All sections | | 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | All sections | diff --git a/etc.txt b/etc.txt index 4de8893..eea4bfd 100644 --- a/etc.txt +++ b/etc.txt @@ -1,7 +1,7 @@ #!/usr/bin/env julia # Test script for mixed-content message testing # Tests receiving a mix of text, json, table, image, audio, video, and binary data -# from Julia serviceA to Julia serviceB using msghandler.jl smartreceive +# from Julia serviceA to Julia serviceB using msghandler.jl smartunpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. @@ -38,9 +38,9 @@ function test_mix_receive() log_trace("Received message on $(msg.subject)") incoming_msg = msg - # # Use msghandler.smartreceive to handle the data - # # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - # result = msghandler.smartreceive( + # # Use msghandler.smartunpack to handle the data + # # API: smartunpack(msg, download_handler; max_retries, base_delay, max_delay) + # result = msghandler.smartunpack( # msg; # max_retries = 5, # base_delay = 100, @@ -229,7 +229,7 @@ println("Note: This receiver will wait for messages from the sender.") println("Run test_julia_to_julia_mix_sender.jl first to send test data.") # Run receiver -println("\ntesting smartreceive for mixed content") +println("\ntesting smartunpack for mixed content") incoming_msg = test_mix_receive() println("\nTest completed.") @@ -250,7 +250,7 @@ println("\nTest completed.") Check architecture.md. For sending table I want to add JSON in addition to Apache Arrow. Currently I use "table" datatype when sending table data using Arrow. Now table that I want to send using JSON I will use "jsontable" as datatype while sending table using Arrow I will use "arrowtable" as datatype. -This will select how smartsend and smartreceive serialize/deserialize the table. +This will select how smartpack and smartunpack serialize/deserialize the table. Can you help me do this? Save the updated architecture.md into updated_architecture.md file. I will deal with source code later. diff --git a/examples/smartreceive_example.rs b/examples/smartreceive_example.rs index 56a36fd..7ae95b1 100644 --- a/examples/smartreceive_example.rs +++ b/examples/smartreceive_example.rs @@ -1,4 +1,4 @@ -use msghandler::{smartreceive, SmartreceiveOptions}; +use msghandler::{smartunpack, smartunpackOptions}; fn main() { // Simulated message JSON (received via any transport) @@ -40,9 +40,9 @@ fn main() { ] }"#; - let options = SmartreceiveOptions::default(); + let options = smartunpackOptions::default(); - match smartreceive(msg_json_str, &options) { + match smartunpack(msg_json_str, &options) { Ok(envelope) => { println!("=== Envelope Received ==="); println!("Correlation ID: {}", envelope.correlation_id); diff --git a/examples/smartsend_example.rs b/examples/smartsend_example.rs index e145b1f..8d244f1 100644 --- a/examples/smartsend_example.rs +++ b/examples/smartsend_example.rs @@ -1,4 +1,4 @@ -use msghandler::{smartsend, Payload, SmartsendOptions}; +use msghandler::{smartpack, Payload, smartpackOptions}; fn main() { // Create mixed payload data @@ -24,7 +24,7 @@ fn main() { ), ]; - let options = SmartsendOptions { + let options = smartpackOptions { broker_url: "localhost:4222".to_string(), fileserver_url: "http://localhost:8080".to_string(), msg_purpose: "chat".to_string(), @@ -32,7 +32,7 @@ fn main() { ..Default::default() }; - match smartsend("/agent/wine/api/v1/prompt", &payloads, &options) { + match smartpack("/agent/wine/api/v1/prompt", &payloads, &options) { Ok((envelope, json_str)) => { println!("=== Envelope Created ==="); println!("Correlation ID: {}", envelope.correlation_id); diff --git a/src/msghandler-csr.js b/src/msghandler-csr.js index 9c07a3c..5b075ce 100644 --- a/src/msghandler-csr.js +++ b/src/msghandler-csr.js @@ -378,13 +378,13 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * * @example * // Send a single payload - * const [env, envJsonStr] = await msghandlerCSR.smartsend( + * const [env, envJsonStr] = await msghandlerCSR.smartpack( * "/test", * [["dataname1", data1, "dictionary"]] * ); * * // Send multiple payloads (use jsontable instead of arrowtable for browser) - * const [env, envJsonStr] = await msghandlerCSR.smartsend( + * const [env, envJsonStr] = await msghandlerCSR.smartpack( * "/test", * [ * ["dataname1", data1, "dictionary"], @@ -395,7 +395,7 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * // Publish via your transport (NATS, MQTT, HTTP, etc.) * // await myNatsClient.publish("/test", envJsonStr); */ -async function smartsend(subject, data, options = {}) { +async function smartpack(subject, data, options = {}) { const { broker_url = DEFAULT_BROKER_URL, fileserver_url = DEFAULT_FILESERVER_URL, @@ -412,20 +412,20 @@ async function smartsend(subject, data, options = {}) { sender_id = uuidv4() } = options; - logTrace(correlation_id, `Starting smartsend for subject: ${subject}`); - logTrace(correlation_id, `smartsend: data array length=${data.length}`); + logTrace(correlation_id, `Starting smartpack for subject: ${subject}`); + logTrace(correlation_id, `smartpack: data array length=${data.length}`); // Debug: Log input data structure for (let i = 0; i < data.length; i++) { const [dataname, payloadData, payloadType] = data[i]; - logTrace(correlation_id, `smartsend: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); } // Process payloads const payloads = []; for (const [dataname, payloadData, payloadType] of data) { - logTrace(correlation_id, `smartsend: Processing payload '${dataname}' type=${payloadType}`); - logTrace(correlation_id, `smartsend: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: Processing payload '${dataname}' type=${payloadType}`); + logTrace(correlation_id, `smartpack: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); const payloadBytes = await serializeData(payloadData, payloadType); const payloadSize = payloadBytes.byteLength; @@ -502,7 +502,7 @@ async function smartsend(subject, data, options = {}) { * * @example * // Receive from JSON string directly - * const env = await msghandlerCSR.smartreceive(jsonString, { + * const env = await msghandlerCSR.smartunpack(jsonString, { * fileserver_download_handler: msghandlerCSR.fetchWithBackoff, * max_retries: 5, * base_delay: 100, @@ -510,7 +510,7 @@ async function smartsend(subject, data, options = {}) { * }); * * // Receive from transport message object (e.g., NATS, MQTT) - * const env = await msghandlerCSR.smartreceive(natsMsg, { + * const env = await msghandlerCSR.smartunpack(natsMsg, { * fileserver_download_handler: msghandlerCSR.fetchWithBackoff * }); * // env.payloads is an Array of [dataname, data, type] arrays @@ -518,7 +518,7 @@ async function smartsend(subject, data, options = {}) { * console.log(`${dataname}: ${data} (type: ${type})`); * } */ -async function smartreceive(msg, options = {}) { +async function smartunpack(msg, options = {}) { const { fileserver_download_handler = fetchWithBackoff, max_retries = 5, @@ -542,28 +542,28 @@ async function smartreceive(msg, options = {}) { throw new Error('Invalid message format: expected JSON string or message object'); } - logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); + logTrace('smartunpack', `smartunpack: raw payload length=${payload.length}`); // Debug: Show first 200 chars of payload const payloadPreview = payload.substring(0, 200); - logTrace('smartreceive', `smartreceive: payload preview: ${payloadPreview}`); + logTrace('smartunpack', `smartunpack: payload preview: ${payloadPreview}`); let envJsonObj; try { envJsonObj = JSON.parse(payload); } catch (e) { - logTrace('smartreceive', `smartreceive: JSON parse failed: ${e.message}`); + logTrace('smartunpack', `smartunpack: JSON parse failed: ${e.message}`); throw e; } logTrace(envJsonObj.correlation_id, 'Processing received message'); - logTrace(envJsonObj.correlation_id, `smartreceive: envelope has ${envJsonObj.payloads.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: envelope has ${envJsonObj.payloads.length} payloads`); // Process all payloads in the envelope const payloadsList = []; const numPayloads = envJsonObj.payloads.length; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing ${numPayloads} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing ${numPayloads} payloads`); for (let i = 0; i < numPayloads; i++) { const payloadObj = envJsonObj.payloads[i]; @@ -571,7 +571,7 @@ async function smartreceive(msg, options = {}) { const dataname = payloadObj.dataname; const payloadType = payloadObj.payload_type; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); if (transport === 'direct') { logTrace(envJsonObj.correlation_id, `Direct transport - decoding payload '${dataname}'`); @@ -614,7 +614,7 @@ async function smartreceive(msg, options = {}) { } } - logTrace(envJsonObj.correlation_id, `smartreceive: Successfully processed all ${payloadsList.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Successfully processed all ${payloadsList.length} payloads`); envJsonObj.payloads = payloadsList; return envJsonObj; } @@ -625,12 +625,12 @@ const msghandlerCSR = { /** * Send data with automatic transport selection */ - smartsend, + smartpack, /** * Receive and process messages */ - smartreceive, + smartunpack, /** * Upload data to plik server in one-shot mode diff --git a/src/msghandler.jl b/src/msghandler.jl index 859c4d1..87fcc2f 100644 --- a/src/msghandler.jl +++ b/src/msghandler.jl @@ -1,5 +1,5 @@ # Bi-Directional Data Bridge - Julia Module -# Implements smartsend and smartreceive for message transport +# Implements smartpack and smartunpack for message transport # This module provides functionality for sending and receiving data across network boundaries # with support for both direct payload transport and # URL-based transport for larger payloads. @@ -24,10 +24,10 @@ # # API Standard: # ```jldoctest -# # Input format for smartsend (always a list of tuples with type info) +# # Input format for smartpack (always a list of tuples with type info) # [(dataname1, data1, type1), (dataname2, data2, type2), ...] # -# # Output format for smartreceive (always returns a list of tuples) +# # Output format for smartunpack (always returns a list of tuples) # [(dataname1, data1, type1), (dataname2, data2, type2), ...] # ``` # @@ -337,7 +337,7 @@ function log_trace(correlation_id::String, message::String) end -""" smartsend - Send data with automatic transport selection, depending on payload size +""" smartpack - Send data with automatic transport selection, depending on payload size This function intelligently routes data delivery based on payload size relative to a threshold. If the serialized payload is smaller than `size_threshold`, it encodes the data as Base64 and constructs a "direct" msg_payload_v1. @@ -392,23 +392,23 @@ using UUIDs # Send a single payload (still wrapped in a list) data = Dict("key" => "value") -env, msg_json = smartsend("my.subject", [("dataname1", data, "dictionary")]) +env, msg_json = smartpack("my.subject", [("dataname1", data, "dictionary")]) # Send multiple payloads in one message with different types data1 = Dict("key1" => "value1") data2 = rand(10_000) # Small array -env, msg_json = smartsend("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")]) +env, msg_json = smartpack("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")]) # Send a large array using fileserver upload data = rand(10_000_000) # ~80 MB -env, msg_json = smartsend("large.data", [("large_arrow_table", data, "arrowtable")]) +env, msg_json = smartpack("large.data", [("large_arrow_table", data, "arrowtable")]) # Send jsontable (JSON format) rows = [Dict("id" => 1, "name" => "Alice"), Dict("id" => 2, "name" => "Bob")] -env, msg_json = smartsend("json.data", [("users", rows, "jsontable")]) +env, msg_json = smartpack("json.data", [("users", rows, "jsontable")]) # Mixed content (e.g., chat with text and image) -env, msg_json = smartsend("chat.subject", [ +env, msg_json = smartpack("chat.subject", [ ("message_text", "Hello!", "text"), ("user_image", image_data, "image"), ("audio_clip", audio_data, "audio") @@ -419,8 +419,8 @@ env, msg_json = smartsend("chat.subject", [ # my_transport.publish(conn, subject, env_json_str) ``` """ -function smartsend( - subject::String, # smartreceive's subject +function smartpack( + subject::String, # smartunpack's subject data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples. Use Tuple{String, Any, String}[] for empty payloads broker_url::String = DEFAULT_BROKER_URL, # Broker URL fileserver_url = DEFAULT_FILESERVER_URL, @@ -446,7 +446,7 @@ function smartsend( )::Tuple{msg_envelope_v1, String} where {T1<:Any} # Log start of send operation - log_trace(correlation_id, "Starting smartsend for subject: $subject") + log_trace(correlation_id, "Starting smartpack for subject: $subject") # Process each payload in the list payloads = msg_payload_v1[] @@ -772,7 +772,7 @@ end # end -""" smartreceive - Receive and process messages +""" smartunpack - Receive and process messages This function processes incoming messages, handling both direct transport (base64 decoded payloads) and link transport (URL-based payloads). It deserializes the data based on the transport type and returns the result. @@ -801,11 +801,11 @@ A HTTP file server is required along with its download function. ```jldoctest # Receive and process message msg_json_str = String(msg.payload) -env = smartreceive(msg_json_str; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) +env = smartunpack(msg_json_str; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) # env["payloads"] = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] ``` """ -function smartreceive( +function smartunpack( msg_json_str::String; # get it from String(nats_msg.payload) fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, diff --git a/src/msghandler.js b/src/msghandler.js index d8b8fcc..ae97d8b 100644 --- a/src/msghandler.js +++ b/src/msghandler.js @@ -431,13 +431,13 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * * @example * // Send a single payload - * const [env, envJsonStr] = await smartsend( + * const [env, envJsonStr] = await smartpack( * "/test", * [["dataname1", data1, "dictionary"]] * ); * * // Send multiple payloads - * const [env, envJsonStr] = await smartsend( + * const [env, envJsonStr] = await smartpack( * "/test", * [ * ["dataname1", data1, "dictionary"], @@ -448,7 +448,7 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * // Publish via your transport (NATS, MQTT, HTTP, etc.) * // await myNatsClient.publish("/test", envJsonStr); */ -async function smartsend(subject, data, options = {}) { +async function smartpack(subject, data, options = {}) { const { broker_url = DEFAULT_BROKER_URL, fileserver_url = DEFAULT_FILESERVER_URL, @@ -465,20 +465,20 @@ async function smartsend(subject, data, options = {}) { sender_id = uuidv4() } = options; - logTrace(correlation_id, `Starting smartsend for subject: ${subject}`); - logTrace(correlation_id, `smartsend: data array length=${data.length}`); + logTrace(correlation_id, `Starting smartpack for subject: ${subject}`); + logTrace(correlation_id, `smartpack: data array length=${data.length}`); // Debug: Log input data structure for (let i = 0; i < data.length; i++) { const [dataname, payloadData, payloadType] = data[i]; - logTrace(correlation_id, `smartsend: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); } // Process payloads const payloads = []; for (const [dataname, payloadData, payloadType] of data) { - logTrace(correlation_id, `smartsend: Processing payload '${dataname}' type=${payloadType}`); - logTrace(correlation_id, `smartsend: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: Processing payload '${dataname}' type=${payloadType}`); + logTrace(correlation_id, `smartpack: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); const payloadBytes = await serializeData(payloadData, payloadType); const payloadSize = payloadBytes.byteLength; @@ -552,7 +552,7 @@ async function smartsend(subject, data, options = {}) { * * @example * // Receive from JSON string directly - * const env = await smartreceive(jsonString, { + * const env = await smartunpack(jsonString, { * fileserver_download_handler: fetchWithBackoff, * max_retries: 5, * base_delay: 100, @@ -560,7 +560,7 @@ async function smartsend(subject, data, options = {}) { * }); * * // Receive from transport message object (e.g., NATS, MQTT) - * const env = await smartreceive(natsMsg, { + * const env = await smartunpack(natsMsg, { * fileserver_download_handler: fetchWithBackoff * }); * // env.payloads is an Array of [dataname, data, type] arrays @@ -568,7 +568,7 @@ async function smartsend(subject, data, options = {}) { * console.log(`${dataname}: ${data} (type: ${type})`); * } */ -async function smartreceive(msg, options = {}) { +async function smartunpack(msg, options = {}) { const { fileserver_download_handler = fetchWithBackoff, max_retries = 5, @@ -592,28 +592,28 @@ async function smartreceive(msg, options = {}) { throw new Error('Invalid message format: expected JSON string or message object'); } - logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); + logTrace('smartunpack', `smartunpack: raw payload length=${payload.length}`); // Debug: Show first 200 chars of payload const payloadPreview = payload.substring(0, 200); - logTrace('smartreceive', `smartreceive: payload preview: ${payloadPreview}`); + logTrace('smartunpack', `smartunpack: payload preview: ${payloadPreview}`); let envJsonObj; try { envJsonObj = JSON.parse(payload); } catch (e) { - logTrace('smartreceive', `smartreceive: JSON parse failed: ${e.message}`); + logTrace('smartunpack', `smartunpack: JSON parse failed: ${e.message}`); throw e; } logTrace(envJsonObj.correlation_id, 'Processing received message'); - logTrace(envJsonObj.correlation_id, `smartreceive: envelope has ${envJsonObj.payloads.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: envelope has ${envJsonObj.payloads.length} payloads`); // Process all payloads in the envelope const payloadsList = []; const numPayloads = envJsonObj.payloads.length; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing ${numPayloads} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing ${numPayloads} payloads`); for (let i = 0; i < numPayloads; i++) { const payloadObj = envJsonObj.payloads[i]; @@ -621,7 +621,7 @@ async function smartreceive(msg, options = {}) { const dataname = payloadObj.dataname; const payloadType = payloadObj.payload_type; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); if (transport === 'direct') { logTrace(envJsonObj.correlation_id, `Direct transport - decoding payload '${dataname}'`); @@ -664,7 +664,7 @@ async function smartreceive(msg, options = {}) { } } - logTrace(envJsonObj.correlation_id, `smartreceive: Successfully processed all ${payloadsList.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Successfully processed all ${payloadsList.length} payloads`); envJsonObj.payloads = payloadsList; return envJsonObj; } @@ -675,12 +675,12 @@ const msghandler = { /** * Send data with automatic transport selection */ - smartsend, + smartpack, /** * Receive and process messages */ - smartreceive, + smartunpack, /** * Upload data to plik server in one-shot mode diff --git a/src/msghandler.py b/src/msghandler.py index f29f131..032ba36 100644 --- a/src/msghandler.py +++ b/src/msghandler.py @@ -372,7 +372,7 @@ def _build_payload( } -async def smartsend( +async def smartpack( subject: str, data: List[Tuple[str, Any, str]], broker_url: str = DEFAULT_BROKER_URL, @@ -429,7 +429,7 @@ async def smartsend( Example: >>> # Send a single payload (still wrapped in a list) >>> data = {"key": "value"} - >>> env, env_json_str = await smartsend( + >>> env, env_json_str = await smartpack( ... "my.subject", ... [("dataname1", data, "dictionary")] ... ) @@ -444,7 +444,7 @@ async def smartsend( if sender_id is None: sender_id = str(uuid.uuid4()) - log_trace(correlation_id, f"Starting smartsend for subject: {subject}") + log_trace(correlation_id, f"Starting smartpack for subject: {subject}") # Process payloads payloads = [] @@ -494,7 +494,7 @@ async def smartsend( return env, env_json_str -async def smartreceive( +async def smartunpack( msg: Any, fileserver_download_handler: Callable = fetch_with_backoff, max_retries: int = 5, @@ -521,10 +521,10 @@ async def smartreceive( Example: >>> # Receive from JSON string directly - >>> env = await smartreceive(json_string) + >>> env = await smartunpack(json_string) >>> >>> # Receive from transport message object (e.g., NATS, MQTT) - >>> env = await smartreceive(nats_msg, fileserver_download_handler=fetch_with_backoff) + >>> env = await smartunpack(nats_msg, fileserver_download_handler=fetch_with_backoff) >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] >>> for dataname, data, type_ in env["payloads"]: >>> print(f"{dataname}: {data} (type: {type_})") @@ -623,7 +623,7 @@ class msghandler: self.broker_url = broker_url or self.DEFAULT_BROKER_URL self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL - async def smartsend( + async def smartpack( self, subject: str, data: List[Tuple[str, Any, str]], @@ -635,16 +635,16 @@ class msghandler: Args: subject: Subject/topic to send to data: List of (dataname, data, type) tuples - **kwargs: Additional options passed to smartsend + **kwargs: Additional options passed to smartpack Returns: Tuple of (env, env_json_str) """ kwargs['broker_url'] = kwargs.get('broker_url', self.broker_url) kwargs['fileserver_url'] = kwargs.get('fileserver_url', self.fileserver_url) - return await smartsend(subject, data, **kwargs) + return await smartpack(subject, data, **kwargs) - async def smartreceive( + async def smartunpack( self, msg: Any, **kwargs @@ -654,12 +654,12 @@ class msghandler: Args: msg: Message to process - **kwargs: Additional options passed to smartreceive + **kwargs: Additional options passed to smartunpack Returns: Dict with envelope metadata and payloads """ - return await smartreceive(msg, **kwargs) + return await smartunpack(msg, **kwargs) # Convenience functions for module-level usage @@ -679,7 +679,7 @@ def send( Returns: Tuple of (env, env_json_str) """ - return asyncio.run(smartsend(subject, data, **kwargs)) + return asyncio.run(smartpack(subject, data, **kwargs)) def receive( @@ -696,12 +696,12 @@ def receive( Returns: Dict with envelope metadata and payloads """ - return asyncio.run(smartreceive(msg, **kwargs)) + return asyncio.run(smartunpack(msg, **kwargs)) __all__ = [ - 'smartsend', - 'smartreceive', + 'smartpack', + 'smartunpack', 'plik_oneshot_upload', 'fetch_with_backoff', 'msghandler', diff --git a/src/msghandler.rs b/src/msghandler.rs index 102ca27..01801e8 100644 --- a/src/msghandler.rs +++ b/src/msghandler.rs @@ -1,6 +1,6 @@ // msghandler Rust Module // Cross-platform bi-directional data bridge -// Implements smartsend and smartreceive for message transport +// Implements smartpack and smartunpack for message transport // with support for both direct payload transport and URL-based transport // for larger payloads using the Claim-Check pattern. // @@ -325,8 +325,8 @@ impl MsgEnvelopeV1 { // Options Structures // ============================================================================ -/// Options for the `smartsend` function -pub struct SmartsendOptions { +/// Options for the `smartpack` function +pub struct smartpackOptions { /// Broker URL pub broker_url: String, /// HTTP file server URL for large payloads @@ -355,9 +355,9 @@ pub struct SmartsendOptions { pub sender_id: String, } -impl Default for SmartsendOptions { +impl Default for smartpackOptions { fn default() -> Self { - SmartsendOptions { + smartpackOptions { broker_url: DEFAULT_BROKER_URL.to_string(), fileserver_url: DEFAULT_FILESERVER_URL.to_string(), fileserver_upload_handler: None, @@ -375,8 +375,8 @@ impl Default for SmartsendOptions { } } -/// Options for the `smartreceive` function -pub struct SmartreceiveOptions { +/// Options for the `smartunpack` function +pub struct smartunpackOptions { /// Custom file server download handler (optional, uses exponential backoff by default) pub fileserver_download_handler: Option>, /// Maximum retry attempts for fetching a URL @@ -387,9 +387,9 @@ pub struct SmartreceiveOptions { pub max_delay: u64, } -impl Default for SmartreceiveOptions { +impl Default for smartunpackOptions { fn default() -> Self { - SmartreceiveOptions { + smartunpackOptions { fileserver_download_handler: None, max_retries: DEFAULT_MAX_RETRIES, base_delay: DEFAULT_BASE_DELAY, @@ -689,7 +689,7 @@ pub fn log_trace(correlation_id: &str, message: &str) { } // ============================================================================ -// Public API: smartsend +// Public API: smartpack // ============================================================================ /// Send data with automatic transport selection. @@ -715,23 +715,23 @@ pub fn log_trace(correlation_id: &str, message: &str) { /// /// # Example /// ```no_run -/// use msghandler::{smartsend, Payload, SmartsendOptions}; +/// use msghandler::{smartpack, Payload, smartpackOptions}; /// -/// let (envelope, json_str) = smartsend( +/// let (envelope, json_str) = smartpack( /// "/agent/wine/api/v1/prompt", /// &[ /// ("msg".to_string(), Payload::Text("Hello!".to_string()), "text".to_string()), /// ("data".to_string(), Payload::Binary(vec![1, 2, 3]), "binary".to_string()), /// ], -/// &SmartsendOptions::default(), +/// &smartpackOptions::default(), /// ).unwrap(); /// /// // Caller publishes via their preferred transport /// ``` -pub fn smartsend( +pub fn smartpack( subject: &str, data: &[(String, Payload, String)], - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { let correlation_id = if options.correlation_id.is_empty() { Uuid::new_v4().to_string() @@ -752,7 +752,7 @@ pub fn smartsend( }; log_trace(&correlation_id, &format!( - "Starting smartsend for subject: {}", subject + "Starting smartpack for subject: {}", subject )); let mut payloads: Vec = Vec::new(); @@ -844,7 +844,7 @@ pub fn smartsend( // ============================================================================ /// Store deserialized Payload data back into a MsgPayloadV1's data field. -/// After smartreceive(), payload.data contains the deserialized content as a string +/// After smartunpack(), payload.data contains the deserialized content as a string /// (decoded text, JSON string, or base64 for binary types). fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> MsgPayloadV1 { let mut p = payload.clone(); @@ -861,7 +861,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms } // ============================================================================ -// Public API: smartreceive +// Public API: smartunpack // ============================================================================ /// Receive and process messages. @@ -880,7 +880,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// /// # Example /// ```no_run -/// use msghandler::{smartreceive, SmartreceiveOptions}; +/// use msghandler::{smartunpack, smartunpackOptions}; /// use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; /// /// let msg_json_str = r#"{"correlation_id":"abc123","msg_id":"msg-uuid", @@ -893,7 +893,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// "data":"SGVsbG8=","metadata":{"payload_bytes":5} /// }]}"#; /// -/// let envelope = smartreceive(msg_json_str, &SmartreceiveOptions::default()).unwrap(); +/// let envelope = smartunpack(msg_json_str, &smartunpackOptions::default()).unwrap(); /// /// for payload in &envelope.payloads { /// if payload.transport == "direct" { @@ -904,9 +904,9 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// } /// } /// ``` -pub fn smartreceive( +pub fn smartunpack( msg_json_str: &str, - options: &SmartreceiveOptions, + options: &smartunpackOptions, ) -> Result { // Parse the JSON envelope let mut env: MsgEnvelopeV1 = serde_json::from_str(msg_json_str) @@ -998,9 +998,9 @@ pub fn smartreceive( pub fn send_text( subject: &str, text: &str, - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { - smartsend( + smartpack( subject, &[( "text".to_string(), @@ -1015,9 +1015,9 @@ pub fn send_text( pub fn send_dictionary( subject: &str, data: &JsonValue, - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { - smartsend( + smartpack( subject, &[( "dictionary".to_string(), @@ -1032,9 +1032,9 @@ pub fn send_dictionary( pub fn send_binary( subject: &str, data: &[u8], - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { - smartsend( + smartpack( subject, &[( "binary".to_string(), @@ -1094,10 +1094,10 @@ pub fn plik_upload_file( // All public types are already exported via `pub` on their definitions. // Key types: -// - `smartsend`, `smartreceive` - main API functions +// - `smartpack`, `smartunpack` - main API functions // - `Payload` - type-safe payload enum // - `MsgEnvelopeV1`, `MsgPayloadV1` - wire format structs -// - `SmartsendOptions`, `SmartreceiveOptions` - configuration +// - `smartpackOptions`, `smartunpackOptions` - configuration // - `FileUploadHandler`, `FileDownloadHandler` - trait abstractions // - `PlikOneshotUploadHandler`, `BackoffDownloadHandler` - default implementations // - `MsgHandlerError` - error type @@ -1193,12 +1193,12 @@ mod tests { #[test] fn test_default_options() { - let opts = SmartsendOptions::default(); + let opts = smartpackOptions::default(); assert_eq!(opts.size_threshold, DEFAULT_SIZE_THRESHOLD); assert_eq!(opts.broker_url, DEFAULT_BROKER_URL); assert_eq!(opts.fileserver_url, DEFAULT_FILESERVER_URL); - let opts = SmartreceiveOptions::default(); + let opts = smartunpackOptions::default(); assert_eq!(opts.max_retries, DEFAULT_MAX_RETRIES); assert_eq!(opts.base_delay, DEFAULT_BASE_DELAY); assert_eq!(opts.max_delay, DEFAULT_MAX_DELAY); diff --git a/src/msghandler_mpy.py b/src/msghandler_mpy.py index e6ec6ab..99fb5db 100644 --- a/src/msghandler_mpy.py +++ b/src/msghandler_mpy.py @@ -263,7 +263,7 @@ def _publish(subject, message, correlation_id): # Placeholder - actual implementation would publish via preferred transport -def smartsend(subject, data, **kwargs): +def smartpack(subject, data, **kwargs): """ Send data with automatic transport selection. @@ -306,7 +306,7 @@ def smartsend(subject, data, **kwargs): Example: >>> # Send text payload - >>> env, env_json_str = smartsend( + >>> env, env_json_str = smartpack( ... "/chat", ... [("message", "Hello!", "text")] ... ) @@ -330,7 +330,7 @@ def smartsend(subject, data, **kwargs): is_publish = kwargs.get('is_publish', True) fileserver_upload_handler = kwargs.get('fileserver_upload_handler', _sync_fileserver_upload) - log_trace(correlation_id, f"Starting smartsend for subject: {subject}") + log_trace(correlation_id, f"Starting smartpack for subject: {subject}") # Process payloads payloads = [] @@ -390,7 +390,7 @@ def smartsend(subject, data, **kwargs): return env, env_json_str -def smartreceive(msg, **kwargs): +def smartunpack(msg, **kwargs): """ Receive and process messages. @@ -414,10 +414,10 @@ def smartreceive(msg, **kwargs): Example: >>> # Receive from JSON string - >>> env = smartreceive(json_string) + >>> env = smartunpack(json_string) >>> >>> # Receive from transport message object - >>> env = smartreceive(transport_msg, fileserver_download_handler=_sync_fileserver_download) + >>> env = smartunpack(transport_msg, fileserver_download_handler=_sync_fileserver_download) >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] >>> for dataname, data, type_ in env["payloads"]: ... print(f"{dataname}: {data} (type: {type_})") @@ -530,34 +530,34 @@ class msghandler: self.broker_url = broker_url or self.DEFAULT_BROKER_URL self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL - def smartsend(self, subject, data, **kwargs): + def smartpack(self, subject, data, **kwargs): """ Send data. Args: subject: Subject/topic to send to data: List of (dataname, data, type) tuples - **kwargs: Additional options passed to smartsend + **kwargs: Additional options passed to smartpack Returns: Tuple of (env, env_json_str) """ kwargs['broker_url'] = kwargs.get('broker_url', self.broker_url) kwargs['fileserver_url'] = kwargs.get('fileserver_url', self.fileserver_url) - return smartsend(subject, data, **kwargs) + return smartpack(subject, data, **kwargs) - def smartreceive(self, msg, **kwargs): + def smartunpack(self, msg, **kwargs): """ Receive and process message. Args: msg: Message to process - **kwargs: Additional options passed to smartreceive + **kwargs: Additional options passed to smartunpack Returns: Dict with envelope metadata and payloads """ - return smartreceive(msg, **kwargs) + return smartunpack(msg, **kwargs) # Convenience functions for module-level usage @@ -573,7 +573,7 @@ def send(subject, data, **kwargs): Returns: Tuple of (env, env_json_str) """ - return smartsend(subject, data, **kwargs) + return smartpack(subject, data, **kwargs) def receive(msg, **kwargs): @@ -587,12 +587,12 @@ def receive(msg, **kwargs): Returns: Dict with envelope metadata and payloads """ - return smartreceive(msg, **kwargs) + return smartunpack(msg, **kwargs) __all__ = [ - 'smartsend', - 'smartreceive', + 'smartpack', + 'smartunpack', 'msghandler', 'send', 'receive', diff --git a/test/test_js_mix_payloads_receiver.js b/test/test_js_mix_payloads_receiver.js index 7c5b75b..6f79b39 100644 --- a/test/test_js_mix_payloads_receiver.js +++ b/test/test_js_mix_payloads_receiver.js @@ -1,6 +1,6 @@ /** * JavaScript Mix Payloads Receiver Test - * Tests the smartreceive function with mixed payload types + * Tests the smartunpack function with mixed payload types * * This test mirrors test_julia_mix_payloads_receiver.jl and demonstrates that * any combination and any number of mixed content can be received correctly. @@ -50,8 +50,8 @@ async function runTest() { console.log(`Received message on ${msg.subject}`); try { - // Process the message using smartreceive - const envelope = await msghandler.smartreceive(msg, { + // Process the message using smartunpack + const envelope = await msghandler.smartunpack(msg, { fileserver_download_handler: msghandler.fetchWithBackoff, max_retries: 5, base_delay: 100, diff --git a/test/test_js_mix_payloads_sender.js b/test/test_js_mix_payloads_sender.js index 7969732..c88eecb 100644 --- a/test/test_js_mix_payloads_sender.js +++ b/test/test_js_mix_payloads_sender.js @@ -1,6 +1,6 @@ /** * JavaScript Mix Payloads Sender Test - * Tests the smartsend function with mixed payload types + * Tests the smartpack function with mixed payload types * * This test mirrors test_julia_mix_payloads_sender.jl and demonstrates that * any combination and any number of mixed content can be sent correctly. @@ -169,7 +169,7 @@ async function runTest() { try { // Send the message console.log('Sending mixed payloads...\n'); - const [env, envJsonStr] = await msghandler.smartsend( + const [env, envJsonStr] = await msghandler.smartpack( TEST_SUBJECT, payloads, { diff --git a/test/test_julia_mix_payloads_receiver.jl b/test/test_julia_mix_payloads_receiver.jl index 673afe8..a88a67b 100644 --- a/test/test_julia_mix_payloads_receiver.jl +++ b/test/test_julia_mix_payloads_receiver.jl @@ -1,7 +1,7 @@ #!/usr/bin/env julia # Test script for mixed-content message testing # Tests receiving a mix of text, json, table, image, audio, video, and binary data -# from Julia serviceA to Julia serviceB using msghandler.jl smartreceive +# from Julia serviceA to Julia serviceB using msghandler.jl smartunpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. @@ -36,9 +36,9 @@ function test_mix_receive() NATS.subscribe(conn, SUBJECT) do msg log_trace("Received message on $(msg.subject)") - # Use msghandler.smartreceive to handle the data - # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - result = msghandler.smartreceive( + # Use msghandler.smartunpack to handle the data + # API: smartunpack(msg, download_handler; max_retries, base_delay, max_delay) + result = msghandler.smartunpack( msg; max_retries = 5, base_delay = 100, @@ -245,7 +245,7 @@ println("Note: This receiver will wait for messages from the sender.") println("Run test_julia_to_julia_mix_sender.jl first to send test data.") # Run receiver -println("\ntesting smartreceive for mixed content") +println("\ntesting smartunpack for mixed content") test_mix_receive() println("\nTest completed.") \ No newline at end of file diff --git a/test/test_julia_mix_payloads_sender.jl b/test/test_julia_mix_payloads_sender.jl index d75b521..a7e77de 100644 --- a/test/test_julia_mix_payloads_sender.jl +++ b/test/test_julia_mix_payloads_sender.jl @@ -1,7 +1,7 @@ #!/usr/bin/env julia # Test script for mixed-content message testing # Tests sending a mix of text, dictionary, arrowtable, jsontable, image, audio, video, and binary data -# from Julia serviceA to Julia serviceB using msghandler.jl smartsend +# from Julia serviceA to Julia serviceB using msghandler.jl smartpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. @@ -166,7 +166,7 @@ function create_sample_data() end -# Sender: Send mixed content via smartsend +# Sender: Send mixed content via smartpack function test_mix_send() # Create sample data (text_data, dict_data, arrow_table_small, arrow_table_large, json_table_small, json_table_large, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data) = create_sample_data() @@ -203,8 +203,8 @@ function test_mix_send() ("binary_file_large", large_binary_data, "binary") ] - # Use smartsend with mixed content - sendinfo = msghandler.smartsend( + # Use smartpack with mixed content + sendinfo = msghandler.smartpack( SUBJECT, payloads; # List of (dataname, data, type) tuples broker_url = NATS_URL, @@ -251,7 +251,7 @@ println("Starting mixed-content transport test...") println("Correlation ID: $correlation_id") # Run sender -println("start smartsend for mixed content") +println("start smartpack for mixed content") test_mix_send() println("\nTest completed.") diff --git a/test/test_py_mix_payloads_sender.py b/test/test_py_mix_payloads_sender.py index 20a9d7f..ea5e798 100644 --- a/test/test_py_mix_payloads_sender.py +++ b/test/test_py_mix_payloads_sender.py @@ -1,6 +1,6 @@ """ Python Mix Payloads Sender Test -Tests the smartsend function with mixed payload types +Tests the smartpack function with mixed payload types """ import asyncio @@ -11,7 +11,7 @@ import base64 # Add parent directory to path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from msghandler import smartsend, DEFAULT_BROKER_URL, DEFAULT_FILESERVER_URL +from msghandler import smartpack, DEFAULT_BROKER_URL, DEFAULT_FILESERVER_URL TEST_SUBJECT = '/test/mix' TEST_BROKER_URL = os.environ.get('NATS_URL', 'nats://localhost:4222') @@ -56,7 +56,7 @@ async def run_test(): try: # Send the message print('Sending mixed payloads...') - env, env_json_str = await smartsend( + env, env_json_str = await smartpack( TEST_SUBJECT, test_data, broker_url=TEST_BROKER_URL, @@ -164,7 +164,7 @@ async def run_test(): ('audio', bytes([0x46, 0x4C, 0x41, 0x43]), 'audio') ] - chat_env, _ = await smartsend( + chat_env, _ = await smartpack( TEST_SUBJECT, chat_data, broker_url=TEST_BROKER_URL,