From df9012e0eb64c8bd3f72e15cc727f3a4640a2a3a Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 15 May 2026 11:55:41 +0700 Subject: [PATCH 01/14] remove NATS integration --- AI_prompt.md | 58 +--- Cargo.toml | 2 +- examples/smartreceive_example.rs | 4 +- examples/smartsend_example.rs | 4 +- src/{natsbridge_csr.js => msghandler-csr.js} | 346 +++---------------- src/{NATSBridge.jl => msghandler.jl} | 71 ++-- src/{natsbridge_ssr.js => msghandler.js} | 325 +++-------------- src/{natsbridge.py => msghandler.py} | 215 +++--------- src/{natsbridge.rs => msghandler.rs} | 47 ++- src/{natsbridge_mpy.py => msghandler_mpy.py} | 155 +++------ 10 files changed, 254 insertions(+), 973 deletions(-) rename src/{natsbridge_csr.js => msghandler-csr.js} (68%) rename src/{NATSBridge.jl => msghandler.jl} (94%) rename src/{natsbridge_ssr.js => msghandler.js} (71%) rename src/{natsbridge.py => msghandler.py} (78%) rename src/{natsbridge.rs => msghandler.rs} (97%) rename src/{natsbridge_mpy.py => msghandler_mpy.py} (82%) diff --git a/AI_prompt.md b/AI_prompt.md index 3e9998a..7f0fc22 100644 --- a/AI_prompt.md +++ b/AI_prompt.md @@ -1,25 +1,16 @@ Consider the following scenarios: -Scenario 1: The "Command & Control" Loop (Low Latency)Focus: Small payloads, Core NATS, bi-directional JSON.The Action: A user on a JavaScript dashboard clicks a "Start Simulation" button. This sends a JSON configuration (parameters like step_size and iterations) to Julia.The Flow: * JS (Sender): Recognizes the message is small ($< 10KB$). Packages it as a direct transport JSON envelope.Julia (Receiver): Listens on the NATS subject, decodes the JSON, and immediately acknowledges receipt with a "Running" status.Project Requirement Met: Fast, low-overhead communication for control signals without involving the fileserver. -Scenario 2: The "Deep Dive" Analysis (High Bandwidth)Focus: Large Arrow tables, Claim-Check pattern, Julia to JS.The Action: Julia finishes a heavy computation and produces a 500MB DataFrame with 10 million rows. It needs to send this to the JS frontend for visualization (e.g., using Perspective.js or D3).The Flow:Julia (Sender): Converts the DataFrame to an Arrow IPC stream. It sees the size is $> 1MB$, so it uploads the bytes to the HTTP fileserver. It then publishes a NATS message with transport: "link" and the URL.JS (Receiver): Receives the URL, fetches the data via fetch(), and uses tableFromIPC() to load the data into memory with zero-copy.Project Requirement Met: Handling massive datasets that exceed NATS message limits while maintaining data integrity across languages. -Scenario 3: Live Audio/Signal Processing (Multimedia & Metadata)Focus: Raw binary, bi-directional streaming, NATS Headers.The Action: The JS client captures a 2-second "chunk" of microphone audio. It needs Julia to perform a Fast Fourier Transform (FFT) or AI transcription.The Flow:JS (Sender): Sends the raw binary WAV/PCM data. It uses NATS Headers to store the metadata ($fs = 44.1kHz$, $channels = 1$) to keep the payload purely binary.Julia (Receiver): Processes the audio and sends back a JSON result (the transcription) and an Arrow Table (the frequency spectrum data).Project Requirement Met: Bi-directional flow involving mixed media (Audio) and technical results (Arrow). -Scenario 4: The "Catch-Up" (Persistence & JetStream)Focus: NATS JetStream, late-joining consumers, state sync.The Action: Julia is constantly publishing "System Health" updates. The JS dashboard is closed for 10 minutes. When the user re-opens the dashboard, they need to see the last 10 minutes of history.The Flow:NATS (Server): Uses a JetStream with a Limits retention policy.JS (Consumer): Connects and requests a "Replay" from the last 10 minutes. It receives a mix of direct (small updates) and link (historical snapshots) messages.Project Requirement Met: Temporal decoupling—consumers can receive data that was sent while they were offline. - -Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service using NATS (Core & JetStream).⚠️ STRICT ARCHITECTURAL CONSTRAINTS (Non-Negotiable)Transport Strategy (Claim-Check Pattern):Direct Path: If payload is < 1MB, send data directly via NATS inside the message envelope (Base64 encoded).Link Path: If payload is > 1MB, upload to a shared HTTP fileserver/store. The NATS message must only contain the metadata and the download URL.Tabular Data Format: * MUST use Apache Arrow IPC Stream for all tables/DataFrames. No CSV or standard JSON-serialization of tables allowed.System Symmetry: * Both services must function as Producers AND Consumers.Modular Elegance: * Implementation must be abstracted into a SmartSend function and a SmartReceive handler. The developer calling these functions should not need to care if the data is going via NATS direct or HTTP link.Technical Stack & Use CasesJulia: NATS.jl, Arrow.jl, JSON3.jl, HTTP.jl.Node.js: nats.js, apache-arrow.Scenarios to Support: * Large Data: Sending a 500MB Arrow table from Julia $\rightarrow$ JS.Media: Sending a 5MB WAV file from JS $\rightarrow$ Julia.Signals: Sending small JSON control commands ($< 10KB$) directly via NATS.Implementation Requirements1. Unified JSON Envelope:Define a schema containing: correlation_id (UUID), type (table/binary/json), transport (direct/link), payload (if direct), and url (if link).2. The Julia Module:Implement SmartSend(subject, data, type): Handles Arrow serialization to an IOBuffer, checks size, and manages HTTP uploads for large blobs.Implement SmartReceive(msg): Parses envelope, handles the HTTP fetch with Exponential Backoff (to avoid race conditions), and restores the DataFrame.Include a basic HTTP.listen server to serve as the temporary storage.3. The JavaScript Module:Implement a symmetric SmartSend using nats.js and apache-arrow.Implement a JetStream Pull Consumer for SmartReceive to ensure backpressure and memory safety.4. Performance & Reliability:Demonstrate "Zero-Copy" reading of the Arrow IPC stream on the JS side.Log the correlation_id at every stage for distributed tracing. - - - - - - - -Create a walkthrough for Julia service-A service sending a mix-content chat message to Julia service-B. the chat message must includes +Scenario 1: The "Command & Control" Loop (Low Latency)Focus: Small payloads, bi-directional JSON.The Action: A user on a JavaScript dashboard clicks a "Start Simulation" button. This sends a JSON configuration (parameters like step_size and iterations) to Julia.The Flow: * JS (Sender): Recognizes the message is small ($< 10KB$). Packages it as a direct transport JSON envelope.Julia (Receiver): Receives the JSON, decodes it, and immediately acknowledges receipt with a "Running" status.Project Requirement Met: Fast, low-overhead communication for control signals without involving the fileserver. +Scenario 2: The "Deep Dive" Analysis (High Bandwidth)Focus: Large Arrow tables, Claim-Check pattern, Julia to JS.The Action: Julia finishes a heavy computation and produces a 500MB DataFrame with 10 million rows. It needs to send this to the JS frontend for visualization (e.g., using Perspective.js or D3).The Flow:Julia (Sender): Converts the DataFrame to an Arrow IPC stream. It sees the size is $> 1MB$, so it uploads the bytes to the HTTP fileserver. It then sends a message with transport: "link" and the URL.JS (Receiver): Receives the URL, fetches the data via fetch(), and uses tableFromIPC() to load the data into memory with zero-copy.Project Requirement Met: Handling massive datasets that exceed message limits while maintaining data integrity across languages. +Scenario 3: Live Audio/Signal Processing (Multimedia & Metadata)Focus: Raw binary, bi-directional streaming, headers for metadata.The Action: The JS client captures a 2-second "chunk" of microphone audio. It needs Julia to perform a Fast Fourier Transform (FFT) or AI transcription.The Flow:JS (Sender): Sends the raw binary WAV/PCM data. It uses transport headers to store the metadata ($fs = 44.1kHz$, $channels = 1$) to keep the payload purely binary.Julia (Receiver): Processes the audio and sends back a JSON result (the transcription) and an Arrow Table (the frequency spectrum data).Project Requirement Met: Bi-directional flow involving mixed media (Audio) and technical results (Arrow). +Scenario 4: The "Catch-Up" (Persistence & State Sync)Focus: Message persistence, late-joining consumers, state sync.The Action: Julia is constantly publishing "System Health" updates. The JS dashboard is closed for 10 minutes. When the user re-opens the dashboard, they need to see the last 10 minutes of history.The Flow:Transport (Server): Uses a persistence layer with a Limits retention policy.JS (Consumer): Connects and requests a "Replay" from the last 10 minutes. It receives a mix of direct (small updates) and link (historical snapshots) messages.Project Requirement Met: Temporal decoupling—consumers can receive data that was sent while they were offline. +Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service, using a unified message envelope with Claim-Check pattern for large payloads.⚠️ STRICT ARCHITECTURAL CONSTRAINTS (Non-Negotiable)Transport Strategy (Claim-Check Pattern):Direct Path: If payload is < 1MB, send data directly inside the message envelope (Base64 encoded).Link Path: If payload is > 1MB, upload to a shared HTTP fileserver/store. The message must only contain the metadata and the download URL.Tabular Data Format: * MUST use Apache Arrow IPC Stream for all tables/DataFrames. No CSV or standard JSON-serialization of tables allowed.System Symmetry: * Both services must function as Producers AND Consumers.Modular Elegance: * Implementation must be abstracted into a SmartSend function and a SmartReceive handler. The developer calling these functions should not need to care if the data is going via direct or HTTP link.Technical Stack & Use CasesJulia: Arrow.jl, JSON3.jl, HTTP.jl.Node.js: apache-arrow, native fetch.Scenarios to Support: * Large Data: Sending a 500MB Arrow table from Julia $\rightarrow$ JS.Media: Sending a 5MB WAV file from JS $\rightarrow$ Julia.Signals: Sending small JSON control commands ($< 10KB$) directly in the envelope.Implementation Requirements1. Unified JSON Envelope:Define a schema containing: correlation_id (UUID), type (table/binary/json), transport (direct/link), payload (if direct), and url (if link).2. The Julia Module:Implement SmartSend(subject, data, type): Handles Arrow serialization to an IOBuffer, checks size, and manages HTTP uploads for large blobs.Implement SmartReceive(msg): Parses envelope, handles the HTTP fetch with Exponential Backoff (to avoid race conditions), and restores the DataFrame.Include a basic HTTP.listen server to serve as the temporary storage.3. The JavaScript Module:Implement a symmetric SmartSend using native fetch and apache-arrow.Implement a JetStream P... (line truncated to 2000 chars) I updated the following: -- msghandler.jl. Essentially I add NATS_connection keyword and new publish_message function to support the keyword. +- msghandler.jl. Essentially I add transport_connection keyword and new publish_message function to support the keyword. Use them and ONLY them as ground truth. Then update the following files accordingly: - architecture.md @@ -30,19 +21,12 @@ All API should be semantically consistent and naming should be consistent across - - - - - Task: Update msghandler.js to reflect recent changes in msghandler.jl and docs Context: msghandler.jl and docs has been updated. Requirements: Source of Truth: Treat the updated msghandler.jl and docs as the definitive source. API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. -Ecosystem Variance: Low-level native functions (e.g., NATS.connect(), JSON.read()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. - - +Ecosystem Variance: Low-level native functions (e.g., connect(), JSON.parse()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. @@ -67,8 +51,6 @@ Now do the following: - - I'm expanding this Julia package (msghandler) into a cross-platform project by adding a JavaScript, Python and MicroPython implementation. The following will serve as the ground truth: @@ -91,9 +73,12 @@ Now, help me do the following: + + + # ---------------------------------------------- 100 --------------------------------------------- # -Got it — let’s rebuild your table in my own teaching style, keeping it crisp, intuitive, and easy for students to grasp. I’ll emphasize **purpose, audience, format, example, and KPI** in a way that flows like a story of how projects move from idea → contract → design → code → review → operations. +Got it — let's rebuild your table in my own teaching style, keeping it crisp, intuitive, and easy for students to grasp. I'll emphasize **purpose, audience, format, example, and KPI** in a way that flows like a story of how projects move from idea → contract → design → code → review → operations. --- @@ -101,42 +86,35 @@ Got it — let’s rebuild your table in my own teaching style, keeping it crisp | Document | Purpose (Rationale) | Primary Audience | Format / Content | Example (SaaS Context) | Measurement (KPI) | |-----------------|---------------------|-----------------|------------------|------------------------|-------------------| -| **Requirements** | Capture the **business intent** — why we’re building this and what success looks like. Defines boundaries and user‑visible outcomes. | Stakeholders, Product Owners, Lead Developers | User stories, PRDs, acceptance criteria, non‑functional constraints. | “System must process tabular data from Julia to SvelteKit UI with <200ms latency for 5‑member teams.” | 95% of requests complete <200ms (synthetic monitoring). | -| **Specification** | The **technical contract** — precise rules for inputs, outputs, and data shape. Ensures consistency across dev and test. | Developers, QA Engineers, CI/CD pipelines | OpenAPI, Protobuf, AsyncAPI. Endpoint definitions, schemas, error codes. | `contract.yaml` defining a NATS subject that accepts Arrow streams with snake_case headers. | 100% of messages validated against spec (CI block rate). | +| **Requirements** | Capture the **business intent** — why we're building this and what success looks like. Defines boundaries and user‑visible outcomes. | Stakeholders, Product Owners, Lead Developers | User stories, PRDs, acceptance criteria, non‑functional constraints. | "System must process tabular data from Julia to SvelteKit UI with <200ms latency for 5‑member teams." | 95% of requests complete <200ms (synthetic monitoring). | +| **Specification** | The **technical contract** — precise rules for inputs, outputs, and data shape. Ensures consistency across dev and test. | Developers, QA Engineers, CI/CD pipelines | OpenAPI, Protobuf, AsyncAPI. Endpoint definitions, schemas, error codes. | `contract.yaml` defining a subject that accepts Arrow streams with snake_case headers. | 100% of messages validated against spec (CI block rate). | | **Architecture** | The **blueprint** — how components fit together, interact, and scale. Guides system structure and trade‑offs. | Architects, Senior Developers, DevOps | C4 diagrams, Mermaid.js, component/network/storage models. | Diagram showing 6‑node cluster routing traffic via Caddy → Node.js API → Julia pods. | 100% of major decisions logged with trade‑off analysis. | -| **Walkthrough** | The **story of flow** — shows how pieces connect end‑to‑end and why steps are sequenced. Builds intuition for new devs. | New Developers, Team Members | TOUR.md, Loom videos, sequence diagrams. Step‑by‑step traces with rationale. | “UI sends JSON → Node.js wraps Claim‑Check → Julia pulls Arrow data (prevents NATS overflow).” | New developers ship feature in <2 days (PR timeline). | +| **Walkthrough** | The **story of flow** — shows how pieces connect end‑to‑end and why steps are sequenced. Builds intuition for new devs. | New Developers, Team Members | TOUR.md, Loom videos, sequence diagrams. Step‑by‑step traces with rationale. | "UI sends JSON → Node.js wraps Claim‑Check → Julia pulls Arrow data (prevents overflow)." | New developers ship feature in <2 days (PR timeline). | | **Implementation** | The **real code** — business logic, helpers, tests, configs. Where design becomes executable. | Developers, Code Reviewers | Source code, README.md, unit tests, setup scripts. | Julia function for matrix calculation + SvelteKit component rendering table. | >80% unit test coverage, <5% drift from spec. | | **Validation** | The **enforcer** — ensures implementation matches the spec. Blocks drift and human error. | Automation servers, QA, Lead Developers | CI jobs, contract tests, linting, integration checks. | CI job rejects PR with camelCase field not allowed by YAML spec. | <1% of PRs bypass validation gates. | | **Runbook** | The **operational manual** — how the system lives in production, scales, and recovers. Guides on‑call engineers. | DevOps, SREs, On‑call Developers | K8s manifests, Helm charts, Markdown guides. Deployment, scaling, backup/restore, troubleshooting. | GitOps manifest ensuring 6 Julia replicas restart if memory >80%. | MTTR <15 minutes for P1 incidents. | - - - - - # ---------------------------------------------- 100 --------------------------------------------- # SDD + GitOps Documentation Stack Document,"Purpose (The ""Rationale"")",Primary Audience,Format / Content,Example (SaaS Context),"Measurement (KPI)" Requirements,"Defines the ""Why"" and the Business Boundary. It sets the constraints and success criteria so the team knows when a feature is ""done"" from a user's perspective.","Stakeholders, Product Owners, Lead Developers","Format: User Stories, PRDs. Content: Functional goals, non-functional requirements (latency, scale), and explicit ""out-of-scope"" items.","""The system must process high-volume tabular data from Julia to the SvelteKit UI with <200ms latency for 5-member teams."",""Pass/Fail: 95% of requests complete <200ms (measured via synthetic monitoring)"" -The Spec,"The Technical Contract. It serves as the single source of truth that defines the shape of data. In SDD, this file drives code generation and automated testing.","Developers, QA Engineers, CI/CD Pipelines","Format: OpenAPI (YAML), Protobuf, AsyncAPI. Content: Endpoint definitions, strict data types, error codes, and request/response schemas.",A contract.yaml defining a NATS subject that accepts an Apache Arrow stream with specific snake_case headers.",""Schema Validation Rate: 100% of messages validated against spec (CI block rate)"" +The Spec,"The Technical Contract. It serves as the single source of truth that defines the shape of data. In SDD, this file drives code generation and automated testing.","Developers, QA Engineers, CI/CD Pipelines","Format: OpenAPI (YAML), Protobuf, AsyncAPI. Content: Endpoint definitions, strict data types, error codes, and request/response schemas.",A contract.yaml defining a subject that accepts an Apache Arrow stream with specific snake_case headers.",""Schema Validation Rate: 100% of messages validated against spec (CI block rate)"" Architecture,"The Structural Blueprint. It explains how the ""pieces"" are arranged in the cluster. It defines the relationships between services, databases, and external providers.","System Architects, Senior Developers, DevOps","Format: C4 Model Diagrams, Mermaid.js. Content: Component diagrams, network flow, storage strategy, and technology stack definitions.",A diagram showing how the 6-node cluster routes traffic through Caddy to the Node.js API and offloads heavy math to Julia pods.",""Architecture Decision Log: 100% of major decisions documented with trade-off analysis"" -Walkthrough,"The Intuition & Flow. It connects multiple APIs/services into a cohesive end-to-end story. It explains the ""steps"" and the ""rationale"" behind the sequence of operations.","New Developers, Current Team Members","Format: TOUR.md, Loom videos, Sequence Diagrams. Content: Step-by-step trace of a feature, explanation of state changes, and the ""why"" behind complex logic.","""End-to-End Trace:"" 1. UI sends JSON to Node.js. 2. Node.js wraps it in a Claim-Check. 3. Julia pulls the Arrow data. Rationale: This prevents NATS memory overflow.",""Onboarding Velocity: New developers deploy feature in <2 days (tracked via PR timeline)"" +Walkthrough,"The Intuition & Flow. It connects multiple APIs/services into a cohesive end-to-end story. It explains the ""steps"" and the ""rationale"" behind the sequence of operations.","New Developers, Current Team Members","Format: TOUR.md, Loom videos, Sequence Diagrams. Content: Step-by-step trace of a feature, explanation of state changes, and the ""why"" behind complex logic.","""End-to-End Trace:"" 1. UI sends JSON to Node.js. 2. Node.js wraps it in a Claim-Check. 3. Julia pulls the Arrow data. Rationale: This prevents overflow.",""Onboarding Velocity: New developers deploy feature in <2 days (tracked via PR timeline)"" Implementation,"The Functional Reality. This is the actual execution of the logic. In SDD, parts of this are auto-generated to ensure it never drifts from the Spec.","Developers, Code Reviewers","Format: Source Code (Git), README.md. Content: Business logic, internal helper functions, unit tests, and local setup instructions.",The Julia function that performs the matrix calculation and the SvelteKit component that renders the resulting table.",""Code Coverage: >80% unit test coverage, <5% test drift from spec"" Validation,"The Enforcement Layer. It ensures that the ""Reality"" (Code) actually matches the ""Contract"" (Spec). It prevents human error from breaking the system.","Automation Servers, QA, Lead Developers","Format: GitHub Actions, Dredd, Prism. Content: Contract tests, linting rules, and integration tests that check API compliance.",A CI job that blocks a Pull Request because a developer added a camelCase field that isn't allowed in the shared YAML spec.",""Block Rate: <1% of PRs reach production without validation (CI gate pass rate)"" Runbook,"The Operational Life-Support. It defines how the system lives in production and how to fix it. In GitOps, the ""State"" is declared here.","DevOps, SREs, On-call Developers","Format: K8s Manifests, Helm Charts, Markdown. Content: Deployment steps, scaling triggers, backup/restore commands, and troubleshooting guides.",A GitOps manifest in Flux that ensures 6 replicas of the Julia service are always running and restarts them if memory hits 80%.",""MTTR: <15 minutes for P1 incidents (tracked via incident management system)"" Do you understand the provided text? Don't fucking change the table content. I want you to add "Measurement (KPI)" column. it is only example of course. This table will be used for consult and teaching. - # ---------------------------------------------- 100 --------------------------------------------- # Can you write the table and explain this approach and each doc in details then save to docs/SDD_FRAMEWORK.md so I can consult it later. Don't forget to add How to use this approach effectively. - # ---------------------------------------------- 100 --------------------------------------------- # Since I develop src folder before I adopt SDD_FRAMEWORK.md approach, can you check src folder and my current doc files then write docs/requirements.md according to SDD framework? Treat src as ground truth. @@ -170,8 +148,6 @@ Can you update the content of the following files according to /home/ton/docker- - - I updated ./src/msghandler.jl. Use it as groundtruth. Check ./docs folder I want to update the content of the following files according to /home/ton/docker-apps/sommpanion/ASG_Framework/ASG_Framework.md: - ./docs/requirements.md - ./docs/specification.md @@ -180,7 +156,6 @@ I updated ./src/msghandler.jl. Use it as groundtruth. Check ./docs folder I want - Check the following files: - ./docs/requirements.md - ./docs/specification.md @@ -193,6 +168,7 @@ Now help me update Rust implementation of this package at ./src/msghandler.rs. + I want to build a client-side-rendering Dioxus-based chat webapp. Dioxus version 0.7+ should be great. I already populate the current folder for the project. diff --git a/Cargo.toml b/Cargo.toml index cd5e45a..9c2423f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "msghandler" version = "1.2.0" edition = "2021" -description = "Cross-platform bi-directional data bridge for NATS communication" +description = "Cross-platform bi-directional data bridge" [lib] name = "msghandler" diff --git a/examples/smartreceive_example.rs b/examples/smartreceive_example.rs index 36ec424..d5919c6 100644 --- a/examples/smartreceive_example.rs +++ b/examples/smartreceive_example.rs @@ -2,7 +2,7 @@ use msghandler::{smartreceive, SmartreceiveOptions}; #[tokio::main] async fn main() { - // Simulated NATS message JSON (received from NATS subscription) + // Simulated message JSON (received via any transport) let msg_json_str = r#"{ "correlation_id": "abc123-def456-ghi789", "msg_id": "msg-uuid-001", @@ -15,7 +15,7 @@ async fn main() { "receiver_id": "", "reply_to": "/agent/wine/api/v1/response", "reply_to_msg_id": "", - "broker_url": "nats://localhost:4222", + "broker_url": "localhost:4222", "metadata": {}, "payloads": [ { diff --git a/examples/smartsend_example.rs b/examples/smartsend_example.rs index ced4d1e..aade6a1 100644 --- a/examples/smartsend_example.rs +++ b/examples/smartsend_example.rs @@ -26,7 +26,7 @@ async fn main() { ]; let options = SmartsendOptions { - broker_url: "nats://localhost:4222".to_string(), + broker_url: "localhost:4222".to_string(), fileserver_url: "http://localhost:8080".to_string(), msg_purpose: "chat".to_string(), sender_name: "rust-example".to_string(), @@ -60,7 +60,7 @@ async fn main() { } println!(); - println!("=== JSON String for NATS Publishing ==="); + println!("=== JSON String for Transport Publishing ==="); println!("{}", json_str); } Err(e) => { diff --git a/src/natsbridge_csr.js b/src/msghandler-csr.js similarity index 68% rename from src/natsbridge_csr.js rename to src/msghandler-csr.js index 7dc5103..9c07a3c 100644 --- a/src/natsbridge_csr.js +++ b/src/msghandler-csr.js @@ -3,8 +3,7 @@ * Browser-Compatible Implementation (Client-Side Rendering) * * This module provides functionality for sending and receiving data across network boundaries - * using NATS as the message bus, with support for both direct payload transport and - * URL-based transport for larger payloads. + * with support for both direct payload transport and URL-based transport for larger payloads. * * Supported payload types: "text", "dictionary", "jsontable", "image", "audio", "video", "binary" * Note: Browser version does NOT support Apache Arrow IPC (arrowtable) due to browser compatibility constraints. @@ -14,10 +13,8 @@ * - Modern browser with ES module support (or use module bundler) * - Web Crypto API for UUID generation * - Fetch API for HTTP requests - * - WebSocket support for NATS connections (use ws:// or wss:// URLs) * * Browser-compatible version uses: - * - nats.ws for WebSocket-based NATS connections * - Web Crypto API for UUID generation * - Uint8Array instead of Buffer * - fetch API for file server communication @@ -25,9 +22,6 @@ * @module msghandlerCSR */ -// Import browser-compatible NATS client -import * as nats from 'nats.ws'; - // Use native fetch available in browsers // ---------------------------------------------- Constants ---------------------------------------------- // @@ -38,9 +32,9 @@ import * as nats from 'nats.ws'; const DEFAULT_SIZE_THRESHOLD = 500_000; /** - * Default NATS server URL (WebSocket protocol) + * Default broker URL */ -const DEFAULT_BROKER_URL = 'ws://localhost:4222'; +const DEFAULT_BROKER_URL = 'localhost:4222'; /** * Default HTTP file server URL for link transport @@ -75,34 +69,6 @@ function base64ToBuffer(base64) { return bytes; } -/** - * Convert Uint8Array to Base64 string (Unicode-safe version) - * Uses TextEncoder/TextDecoder for proper Unicode handling - * @param {Uint8Array} data - Data to encode - * @returns {string} Base64 encoded string - */ -function bufferToBase64UnicodeSafe(data) { - const bytes = new Uint8Array(data); - // Use TextDecoder to properly handle the bytes as text - const binary = String.fromCharCode(...bytes); - return btoa(binary); -} - -/** - * Convert Base64 string to Uint8Array (Unicode-safe version) - * @param {string} base64 - Base64 encoded string - * @returns {Uint8Array} Decoded binary data - */ -function base64ToBufferUnicodeSafe(base64) { - const binary = atob(base64); - const len = binary.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binary.charCodeAt(i); - } - return bytes; -} - /** * Generate UUID v4 using Web Crypto API * @returns {string} UUID string @@ -323,191 +289,11 @@ async function fetchWithBackoff(url, maxRetries, baseDelay, maxDelay, correlatio throw new Error(`Failed to fetch data after ${maxRetries} attempts`); } -// ---------------------------------------------- NATS Client ---------------------------------------------- // - -/** - * NATS client wrapper for connection management - * Supports both single-use and persistent connection modes - */ -class NATSClient { - /** - * Create a new NATS client - * @param {string} url - NATS server URL (ws:// or wss://) - * @param {boolean} [keepAlive=false] - Keep connection open for multiple publishes - */ - constructor(url, keepAlive = false) { - this.url = url; - this.connection = null; - this.keepAlive = keepAlive; - } - - /** - * Connect to NATS server - * @returns {Promise} - */ - async connect() { - if (this.connection) { - return this.connection; - } - this.connection = await nats.connect({ servers: this.url }); - return this.connection; - } - - /** - * Publish message to NATS subject - * @param {string} subject - NATS subject to publish to - * @param {string} message - Message to publish - * @param {string} correlationId - Correlation ID for logging - */ - async publish(subject, message, correlationId) { - if (!this.connection) { - await this.connect(); - } - await this.connection.publish(subject, message); - logTrace(correlationId, `Message published to ${subject}`); - } - - /** - * Close the NATS connection - */ - async close() { - if (this.connection) { - this.connection.close(); - this.connection = null; - } - } - - /** - * Get the current connection (for external use) - * @returns {NATS.Connection|null} - */ - getConnection() { - return this.connection; - } - - /** - * Check if connected - * @returns {boolean} - */ - isConnected() { - return this.connection !== null; - } -} - -/** - * Connection pool for managing multiple NATS connections - * Useful for applications with multiple concurrent publishers - */ -class NATSConnectionPool { - /** - * Create a new connection pool - * @param {string} url - NATS server URL (ws:// or wss://) - * @param {number} [maxSize=10] - Maximum pool size - */ - constructor(url, maxSize = 10) { - this.url = url; - this.maxSize = maxSize; - this.connections = new Map(); - this.idCounter = 0; - } - - /** - * Get a connection from the pool (or create new) - * @returns {Promise} - */ - async acquire() { - // Try to find an existing idle connection - for (const [id, client] of this.connections) { - if (client.isConnected()) { - return client; - } - } - - // Create new connection if under limit - if (this.connections.size < this.maxSize) { - const id = `conn_${++this.idCounter}`; - const client = new NATSClient(this.url, true); - await client.connect(); - this.connections.set(id, client); - return client; - } - - // Pool exhausted - create new connection (caller should close when done) - const client = new NATSClient(this.url, false); - await client.connect(); - return client; - } - - /** - * Return a connection to the pool - * @param {NATSClient} client - Connection to return - */ - release(client) { - // Only return persistent connections - if (client.keepAlive && client.isConnected()) { - // Connection already in pool, do nothing - return; - } - // Non-persistent connection - close it - client.close(); - } - - /** - * Close all connections in the pool - */ - async closeAll() { - for (const [id, client] of this.connections) { - await client.close(); - } - this.connections.clear(); - } -} - // ---------------------------------------------- Core Functions ---------------------------------------------- // -/** - * Publish message to NATS - * @param {string|NATSClient|NATS.Connection} brokerUrlOrClient - NATS URL, client, or connection - * @param {string} subject - NATS subject to publish to - * @param {string} message - JSON message to publish - * @param {string} correlationId - Correlation ID for tracing - * @param {boolean} [closeConnection=true] - Close connection after publish (set false for persistent connections) - */ -async function publishMessage(brokerUrlOrClient, subject, message, correlationId, closeConnection = true) { - let conn; - let shouldClose = false; - - if (brokerUrlOrClient instanceof NATSClient) { - conn = brokerUrlOrClient; - } else if (brokerUrlOrClient && typeof brokerUrlOrClient.publish === 'function') { - // Create a wrapper for direct connection (duck-typing check for NATS connection) - conn = { - async publish(subj, msg) { - await brokerUrlOrClient.publish(subj, msg); - }, - async close() { - await brokerUrlOrClient.close(); - } - }; - shouldClose = true; - } else { - // String URL - create new client - const client = new NATSClient(brokerUrlOrClient); - conn = client; - shouldClose = true; - } - - await conn.publish(subject, message, correlationId); - - // Only close if explicitly requested and it's a short-lived client - if (shouldClose && closeConnection && conn instanceof NATSClient) { - await conn.close(); - } -} - /** * Build message envelope from payloads and metadata - * @param {string} subject - NATS subject + * @param {string} subject - Subject/topic * @param {Array} payloads - Array of payload objects * @param {Object} options - Envelope metadata options * @returns {Object} Envelope object @@ -560,19 +346,22 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { } /** - * Send data via NATS with automatic transport selection + * Send data with automatic transport selection * * This function intelligently routes data delivery based on payload size. * If the serialized payload is smaller than size_threshold, it encodes the data as Base64 - * and publishes directly over NATS. Otherwise, it uploads the data to a fileserver - * and publishes only the download URL over NATS. + * into a "direct" payload. Otherwise, it uploads the data to a fileserver + * and creates a "link" payload with the URL. * - * @param {string} subject - NATS subject to publish the message to + * Transport publishing is the caller's responsibility. This function returns the + * envelope and its JSON string representation. + * + * @param {string} subject - Subject/topic to send the message to * @param {Array} data - List of [dataname, data, type] tuples to send * - type: "text", "dictionary", "jsontable", "image", "audio", "video", "binary" * - Note: "arrowtable" is NOT supported in browser (use "jsontable" for tabular data) * @param {Object} options - Optional configuration - * @param {string} [options.broker_url=DEFAULT_BROKER_URL] - URL of the NATS server (WebSocket) + * @param {string} [options.broker_url=DEFAULT_BROKER_URL] - Broker URL (for envelope metadata) * @param {string} [options.fileserver_url=DEFAULT_FILESERVER_URL] - URL of the HTTP file server * @param {Function} [options.fileserver_upload_handler=plikOneshotUpload] - Function to handle fileserver uploads * @param {number} [options.size_threshold=DEFAULT_SIZE_THRESHOLD] - Threshold separating direct vs link transport @@ -583,8 +372,6 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * @param {string} [options.receiver_id=""] - UUID of the receiver (empty means broadcast) * @param {string} [options.reply_to=""] - Topic to reply to * @param {string} [options.reply_to_msg_id=""] - Message ID this message is replying to - * @param {boolean} [options.is_publish=true] - Whether to automatically publish the message - * @param {NATSClient|NATS.Connection} [options.nats_connection=null] - Pre-existing NATS connection * @param {string} [options.msg_id=uuidv4()] - Message ID * @param {string} [options.sender_id=uuidv4()] - Sender ID * @returns {Promise<[Object, string]>} Tuple of [env, env_json_str] @@ -593,8 +380,7 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * // Send a single payload * const [env, envJsonStr] = await msghandlerCSR.smartsend( * "/test", - * [["dataname1", data1, "dictionary"]], - * { broker_url: "wss://nats.example.com" } + * [["dataname1", data1, "dictionary"]] * ); * * // Send multiple payloads (use jsontable instead of arrowtable for browser) @@ -603,9 +389,11 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * [ * ["dataname1", data1, "dictionary"], * ["dataname2", tableData, "jsontable"] - * ], - * { broker_url: "wss://nats.example.com" } + * ] * ); + * + * // Publish via your transport (NATS, MQTT, HTTP, etc.) + * // await myNatsClient.publish("/test", envJsonStr); */ async function smartsend(subject, data, options = {}) { const { @@ -620,8 +408,6 @@ async function smartsend(subject, data, options = {}) { receiver_id = '', reply_to = '', reply_to_msg_id = '', - is_publish = true, - nats_connection = null, msg_id = uuidv4(), sender_id = uuidv4() } = options; @@ -695,25 +481,18 @@ async function smartsend(subject, data, options = {}) { const env_json_str = JSON.stringify(env); - if (is_publish) { - if (nats_connection) { - await publishMessage(nats_connection, subject, env_json_str, correlation_id); - } else { - await publishMessage(broker_url, subject, env_json_str, correlation_id); - } - } - return [env, env_json_str]; } /** - * Receive and process NATS message + * Receive and process messages * - * This function processes incoming NATS messages, handling both direct transport + * This function processes incoming messages, handling both direct transport * (base64 decoded payloads) and link transport (URL-based payloads). * It deserializes the data based on the transport type and returns the result. * - * @param {Object} msg - NATS message object with payload property + * @param {string|Object} msg - Message payload. Accepts either a JSON string directly, + * or an object with a `data` or `payload` property containing the JSON string. * @param {Object} options - Optional configuration * @param {Function} [options.fileserver_download_handler=fetchWithBackoff] - Function to handle fileserver downloads * @param {number} [options.max_retries=5] - Maximum retry attempts for fetching URL @@ -722,13 +501,18 @@ async function smartsend(subject, data, options = {}) { * @returns {Promise} Envelope object with processed payloads * * @example - * // Receive and process message - * const env = await msghandlerCSR.smartreceive(msg, { + * // Receive from JSON string directly + * const env = await msghandlerCSR.smartreceive(jsonString, { * fileserver_download_handler: msghandlerCSR.fetchWithBackoff, * max_retries: 5, * base_delay: 100, * max_delay: 5000 * }); + * + * // Receive from transport message object (e.g., NATS, MQTT) + * const env = await msghandlerCSR.smartreceive(natsMsg, { + * fileserver_download_handler: msghandlerCSR.fetchWithBackoff + * }); * // env.payloads is an Array of [dataname, data, type] arrays * for (const [dataname, data, type] of env.payloads) { * console.log(`${dataname}: ${data} (type: ${type})`); @@ -742,20 +526,20 @@ async function smartreceive(msg, options = {}) { max_delay = 5000 } = options; - // Debug: Log message object structure - logTrace('smartreceive', `smartreceive: msg object keys: ${Object.keys(msg).join(', ')}`); - logTrace('smartreceive', `smartreceive: msg.data type: ${typeof msg.data}, constructor: ${msg.data?.constructor?.name}`); - logTrace('smartreceive', `smartreceive: msg.payload type: ${typeof msg.payload}, constructor: ${msg.payload?.constructor?.name}`); - - // Parse the JSON envelope - // NATS.js v2.x uses msg.data instead of msg.payload + // Handle both raw JSON strings and transport message objects let payload; - if (msg.data !== undefined) { - payload = typeof msg.data === 'string' ? msg.data : new TextDecoder().decode(msg.data); - } else if (msg.payload !== undefined) { - payload = typeof msg.payload === 'string' ? msg.payload : new TextDecoder().decode(msg.payload); + if (typeof msg === 'string') { + payload = msg; + } else if (msg !== null && typeof msg === 'object') { + if (msg.data !== undefined) { + payload = typeof msg.data === 'string' ? msg.data : new TextDecoder().decode(msg.data); + } else if (msg.payload !== undefined) { + payload = typeof msg.payload === 'string' ? msg.payload : new TextDecoder().decode(msg.payload); + } else { + throw new Error('Message has neither data nor payload property'); + } } else { - throw new Error('Message has neither data nor payload property'); + throw new Error('Invalid message format: expected JSON string or message object'); } logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); @@ -839,61 +623,15 @@ async function smartreceive(msg, options = {}) { const msghandlerCSR = { /** - * NATS client class for connection management - * Supports both single-use and persistent connection modes - * - * @example - * // Single-use connection (closes after publish) - * const client = new msghandlerCSR.NATSClient("wss://nats.example.com"); - * await msghandlerCSR.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client }); - * await client.close(); - * - * // Persistent connection (keeps connection open) - * const client = new msghandlerCSR.NATSClient("wss://nats.example.com", true); - * await client.connect(); - * await msghandlerCSR.smartsend("/test1", [["msg", "Hello", "text"]], { nats_connection: client, is_publish: false }); - * await msghandlerCSR.publishMessage(client, "/test2", JSON.stringify({msg: "World"}), "trace-id"); - * // Connection remains open for more publishes - * await client.close(); - */ - NATSClient, - - /** - * Connection pool for managing multiple NATS connections - * Useful for applications with multiple concurrent publishers - * - * @example - * const pool = new msghandlerCSR.NATSConnectionPool("wss://nats.example.com", 10); - * const client = await pool.acquire(); - * await msghandlerCSR.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client }); - * pool.release(client); - * await pool.closeAll(); - */ - NATSConnectionPool, - - /** - * Send data via NATS with automatic transport selection + * Send data with automatic transport selection */ smartsend, /** - * Receive and process NATS message + * Receive and process messages */ smartreceive, - /** - * Publish message to NATS - * - * @example - * // Using a persistent connection - * const client = new msghandlerCSR.NATSClient("wss://nats.example.com", true); - * await client.connect(); - * await msghandlerCSR.publishMessage(client, "/subject", JSON.stringify({msg: "Hello"}), "trace-id", false); - * // Connection stays open for more publishes - * await client.close(); - */ - publishMessage, - /** * Upload data to plik server in one-shot mode */ diff --git a/src/NATSBridge.jl b/src/msghandler.jl similarity index 94% rename from src/NATSBridge.jl rename to src/msghandler.jl index 3c6372c..859c4d1 100644 --- a/src/NATSBridge.jl +++ b/src/msghandler.jl @@ -1,7 +1,7 @@ # Bi-Directional Data Bridge - Julia Module -# Implements smartsend and smartreceive for NATS communication +# Implements smartsend and smartreceive for message transport # This module provides functionality for sending and receiving data across network boundaries -# using NATS as the message bus, with support for both direct payload transport and +# with support for both direct payload transport and # URL-based transport for larger payloads. # # File Server Handler Architecture: @@ -48,12 +48,12 @@ using JSON, Arrow, HTTP, UUIDs, Dates, Base64, PrettyPrinting, DataFrames # Constants const DEFAULT_SIZE_THRESHOLD = 500_000 # 0.5MB - threshold for switching from direct to link transport -const DEFAULT_BROKER_URL = "nats://localhost:4222" # Default NATS server URL +const DEFAULT_BROKER_URL = "localhost:4222" # Default broker URL const DEFAULT_FILESERVER_URL = "http://localhost:8080" # Default HTTP file server URL for link transport """ msg_payload_v1 - Internal message payload structure -This structure represents a single payload within a NATS message envelope. +This structure represents a single payload within a message envelope. It supports both direct transport (base64-encoded data) and link transport (URL-based). # Arguments: @@ -141,11 +141,11 @@ end """ msg_envelope_v1 - Internal message envelope structure -This structure represents a complete NATS message envelope containing multiple payloads +This structure represents a complete message envelope containing multiple payloads with metadata for routing, tracing, and message context. # Arguments: - - `send_to::String` - NATS subject/topic to publish the message to (e.g., "/agent/wine/api/v1/prompt") + - `send_to::String` - Subject/topic to send the message to (e.g., "/agent/wine/api/v1/prompt") - `payloads::Vector{msg_payload_v1}` - List of payloads to include in the message # Keyword Arguments: @@ -159,7 +159,7 @@ with metadata for routing, tracing, and message context. - `receiver_id::String = ""` - UUID of the receiver (empty string means broadcast) - `reply_to::String = ""` - Topic where receiver should reply (empty string if no reply expected) - `reply_to_msg_id::String = ""` - Message ID this message is replying to - - `broker_url::String = DEFAULT_BROKER_URL` - NATS broker URL + - `broker_url::String = DEFAULT_BROKER_URL` - Broker URL - `metadata::Dict{String, Any} = Dict{String, Any}()` - Optional message-level metadata # Return: @@ -199,7 +199,7 @@ struct msg_envelope_v1 reply_to::String # sender ask receiver to reply to this topic reply_to_msg_id::String # the message id this message is replying to - broker_url::String # NATS server address + broker_url::String # Broker address metadata::Dict{String, Any} payloads::Vector{msg_payload_v1} # multiple payload store here @@ -244,7 +244,7 @@ end """ envelope_to_json - Convert msg_envelope_v1 to JSON string This function converts the msg_envelope_v1 struct to a JSON string representation, -preserving all metadata and payload information for NATS message publishing. +preserving all metadata and payload information for transport publishing. # Function Workflow: 1. Creates a dictionary with envelope metadata (correlation_id, msg_id, timestamp, etc.) @@ -337,7 +337,7 @@ function log_trace(correlation_id::String, message::String) end -""" smartsend - Send data either directly via NATS or via a fileserver URL, depending on payload size +""" smartsend - Send data with automatic transport selection, depending on payload size This function intelligently routes data delivery based on payload size relative to a threshold. If the serialized payload is smaller than `size_threshold`, it encodes the data as Base64 and constructs a "direct" msg_payload_v1. @@ -347,7 +347,7 @@ The function accepts a list of (dataname, data, type) tuples as input and proces Each payload can have a different type, enabling mixed-content messages (e.g., chat with text, images, audio). This function creates and returns the msg_envelope_v1 and its JSON string representation only. -NATS publishing must be performed by the caller. +Transport publishing must be performed by the caller. # Function Workflow: 1. Iterates through the list of (dataname, data, type) tuples @@ -356,10 +356,10 @@ NATS publishing must be performed by the caller. 4. For small payloads: encodes as Base64, constructs a "direct" msg_payload_v1 5. For large payloads: uploads to the fileserver, constructs a "link" msg_payload_v1 with the URL 6. Constructs msg_envelope_v1 with all payloads and metadata -7. Converts envelope to JSON string and returns (NATS publishing is handled by the caller) +7. Converts envelope to JSON string and returns (transport publishing is handled by the caller) # Arguments: - - `subject::String` - NATS subject to publish the message to + - `subject::String` - Subject/topic to send the message to - `data::AbstractArray{Tuple{String, T1, String}, 1}` - List of (dataname, data, type) tuples to send - `dataname::String` - Name of the payload - `data::T1` - The actual data to send (any type supported by `_serialize_data`) @@ -367,7 +367,7 @@ NATS publishing must be performed by the caller. - No standalone `type` parameter - type is specified per payload # Keyword Arguments: - - `broker_url::String = DEFAULT_BROKER_URL` - URL of the NATS server + - `broker_url::String = DEFAULT_BROKER_URL` - URL of the broker - `fileserver_url = DEFAULT_FILESERVER_URL` - URL of the HTTP file server for large payloads - `fileserver_upload_handler::Function = plik_oneshot_upload` - Function to handle fileserver uploads (must return Dict with "status", "uploadid", "fileid", "url" keys) - `size_threshold::Int = DEFAULT_SIZE_THRESHOLD` - Threshold in bytes separating direct vs link transport @@ -414,15 +414,15 @@ env, msg_json = smartsend("chat.subject", [ ("audio_clip", audio_data, "audio") ]) -# Publish the JSON string directly using NATS (manual publish) -# conn = NATS.connect(broker_url) -# NATS.publish(conn, subject, env_json_str) +# Publish the JSON string directly via your transport (manual publish) +# conn = my_transport.connect(broker_url) +# my_transport.publish(conn, subject, env_json_str) ``` """ function smartsend( subject::String, # smartreceive's subject data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples. Use Tuple{String, Any, String}[] for empty payloads - broker_url::String = DEFAULT_BROKER_URL, # NATS server URL + broker_url::String = DEFAULT_BROKER_URL, # Broker URL fileserver_url = DEFAULT_FILESERVER_URL, fileserver_upload_handler::Function = plik_oneshot_upload, # a function to handle uploading data to specific HTTP fileserver size_threshold::Int = DEFAULT_SIZE_THRESHOLD, @@ -461,7 +461,7 @@ function smartsend( # Decision: Direct vs Link if payload_size < size_threshold # Check if payload is small enough for direct transport - # Direct path - Base64 encode and send via NATS + # Direct path - Base64 encode and include in message envelope payload_b64 = Base64.base64encode(payload_bytes) # Encode bytes as base64 string log_trace(correlation_id, "Using direct transport for $payload_size bytes") # Log transport choice @@ -486,7 +486,7 @@ function smartsend( ) push!(payloads, payload) else - # Link path - Upload to HTTP server, send URL via NATS + # Link path - Upload to HTTP server, include URL in message envelope log_trace(correlation_id, "Using link transport, uploading to fileserver") # Log link transport choice # Upload to HTTP server @@ -703,13 +703,13 @@ function _serialize_data(data::Any, payload_type::String) end -# """ publish_message - Publish message to NATS -# This function publishes a message to a NATS subject with proper +# """ publish_message - Publish message via transport +# This function publishes a message via the transport with proper # connection management and logging. # # Arguments: -# - `broker_url::String` - NATS server URL (e.g., "nats://localhost:4222") -# - `subject::String` - NATS subject to publish to (e.g., "/agent/wine/api/v1/prompt") +# - `broker_url::String` - Broker URL (e.g., "localhost:4222") +# - `subject::String` - Subject to publish to (e.g., "/agent/wine/api/v1/prompt") # - `message::String` - JSON message to publish # - `correlation_id::String` - Correlation ID for tracing and logging @@ -723,8 +723,8 @@ end # # Prepare JSON message # message = "{\"correlation_id\":\"abc123\",\"payload\":\"test\"}" -# # Publish to NATS -# publish_message("nats://localhost:4222", "my.subject", message, "abc123") +# # Publish via transport +# publish_message("localhost:4222", "my.subject", message, "abc123") # ``` # """ # function publish_message(broker_url::String, subject::String, message::String, correlation_id::String) @@ -732,13 +732,13 @@ end # publish_message(conn, subject, message, correlation_id) # end -# """ publish_message - Publish message to NATS using pre-existing connection -# This function publishes a message to a NATS subject using a pre-existing NATS connection, +# """ publish_message - Publish message via transport using pre-existing connection +# This function publishes a message via the transport using a pre-existing connection, # avoiding the overhead of connection establishment. # # Arguments: -# - `conn::NATS.Connection` - Pre-existing NATS connection -# - `subject::String` - NATS subject to publish to (e.g., "/agent/wine/api/v1/prompt") +# - `conn` - Pre-existing connection object with publish/close methods +# - `subject::String` - Subject to publish to (e.g., "/agent/wine/api/v1/prompt") # - `message::String` - JSON message to publish # - `correlation_id::String` - Correlation ID for tracing and logging @@ -759,7 +759,7 @@ end # ``` # # Use Case: -# Use this version when you already have an established NATS connection and want to publish +# Use this version when you already have an established connection and want to publish # multiple messages without the overhead of creating a new connection for each publish. # """ # function publish_message(conn::NATS.Connection, subject::String, message::String, correlation_id::String) @@ -772,21 +772,21 @@ end # end -""" smartreceive - Receive and process messages from NATS -This function processes incoming NATS messages, handling both direct transport +""" smartreceive - Receive and process messages +This function processes incoming messages, handling both direct transport (base64 decoded payloads) and link transport (URL-based payloads). It deserializes the data based on the transport type and returns the result. A HTTP file server is required along with its download function. # Function Workflow: -1. Parses the JSON envelope from the NATS message +1. Parses the JSON envelope from the message 2. Iterates through each payload in the envelope 3. For each payload: determines the transport type (direct or link) 4. For direct transport: decodes Base64 payload and deserializes based on type 5. For link transport: fetches data from URL with exponential backoff, then deserializes # Arguments: - - `msg_json_str::String` - JSON string from NATS message payload (e.g., `String(nats_msg.payload)`) + - `msg_json_str::String` - JSON string from the message payload (e.g., `String(msg.payload)`) # Keyword Arguments: - `fileserver_download_handler::Function = _fetch_with_backoff` - Function to handle downloading data from file server URLs @@ -800,7 +800,6 @@ A HTTP file server is required along with its download function. # Example ```jldoctest # Receive and process message -msg = nats_message # NATS message msg_json_str = String(msg.payload) env = smartreceive(msg_json_str; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) # env["payloads"] = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] diff --git a/src/natsbridge_ssr.js b/src/msghandler.js similarity index 71% rename from src/natsbridge_ssr.js rename to src/msghandler.js index 7dc2b0b..d8b8fcc 100644 --- a/src/natsbridge_ssr.js +++ b/src/msghandler.js @@ -3,21 +3,18 @@ * JavaScript/Node.js Implementation (Desktop/Server-Side) * * This module provides functionality for sending and receiving data across network boundaries - * using NATS as the message bus, with support for both direct payload transport and - * URL-based transport for larger payloads. + * with support for both direct payload transport and URL-based transport for larger payloads. * * Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * * Node.js-specific features: * - Apache Arrow IPC support via apache-arrow - * - TCP NATS connections (nats:// or tls:// URLs) * - Buffer for binary data handling - * - Connection pooling for high-throughput scenarios + * - Native fetch for HTTP operations * * @module msghandler */ -const nats = require('nats'); const crypto = require('crypto'); // Use native fetch available in Node.js 18+ const arrow = require('apache-arrow'); @@ -40,9 +37,9 @@ function uuidv4() { const DEFAULT_SIZE_THRESHOLD = 500_000; /** - * Default NATS server URL + * Default broker URL */ -const DEFAULT_BROKER_URL = 'nats://localhost:4222'; +const DEFAULT_BROKER_URL = 'localhost:4222'; /** * Default HTTP file server URL for link transport @@ -344,191 +341,11 @@ async function fetchWithBackoff(url, maxRetries, baseDelay, maxDelay, correlatio throw new Error(`Failed to fetch data after ${maxRetries} attempts`); } -// ---------------------------------------------- NATS Client ---------------------------------------------- // - -/** - * NATS client wrapper for connection management - * Supports both single-use and persistent connection modes - */ -class NATSClient { - /** - * Create a new NATS client - * @param {string} url - NATS server URL (nats:// or tls://) - * @param {boolean} [keepAlive=false] - Keep connection open for multiple publishes - */ - constructor(url, keepAlive = false) { - this.url = url; - this.connection = null; - this.keepAlive = keepAlive; - } - - /** - * Connect to NATS server - * @returns {Promise} - */ - async connect() { - if (this.connection) { - return this.connection; - } - this.connection = await nats.connect({ servers: this.url }); - return this.connection; - } - - /** - * Publish message to NATS subject - * @param {string} subject - NATS subject to publish to - * @param {string} message - Message to publish - * @param {string} correlationId - Correlation ID for logging - */ - async publish(subject, message, correlationId) { - if (!this.connection) { - await this.connect(); - } - await this.connection.publish(subject, message); - logTrace(correlationId, `Message published to ${subject}`); - } - - /** - * Close the NATS connection - */ - async close() { - if (this.connection) { - this.connection.close(); - this.connection = null; - } - } - - /** - * Get the current connection (for external use) - * @returns {NATS.Connection|null} - */ - getConnection() { - return this.connection; - } - - /** - * Check if connected - * @returns {boolean} - */ - isConnected() { - return this.connection !== null; - } -} - -/** - * Connection pool for managing multiple NATS connections - * Useful for applications with multiple concurrent publishers - */ -class NATSConnectionPool { - /** - * Create a new connection pool - * @param {string} url - NATS server URL (nats:// or tls://) - * @param {number} [maxSize=10] - Maximum pool size - */ - constructor(url, maxSize = 10) { - this.url = url; - this.maxSize = maxSize; - this.connections = new Map(); - this.idCounter = 0; - } - - /** - * Get a connection from the pool (or create new) - * @returns {Promise} - */ - async acquire() { - // Try to find an existing idle connection - for (const [id, client] of this.connections) { - if (client.isConnected()) { - return client; - } - } - - // Create new connection if under limit - if (this.connections.size < this.maxSize) { - const id = `conn_${++this.idCounter}`; - const client = new NATSClient(this.url, true); - await client.connect(); - this.connections.set(id, client); - return client; - } - - // Pool exhausted - create new connection (caller should close when done) - const client = new NATSClient(this.url, false); - await client.connect(); - return client; - } - - /** - * Return a connection to the pool - * @param {NATSClient} client - Connection to return - */ - release(client) { - // Only return persistent connections - if (client.keepAlive && client.isConnected()) { - // Connection already in pool, do nothing - return; - } - // Non-persistent connection - close it - client.close(); - } - - /** - * Close all connections in the pool - */ - async closeAll() { - for (const [id, client] of this.connections) { - await client.close(); - } - this.connections.clear(); - } -} - // ---------------------------------------------- Core Functions ---------------------------------------------- // -/** - * Publish message to NATS - * @param {string|NATSClient|NATS.Connection} brokerUrlOrClient - NATS URL, client, or connection - * @param {string} subject - NATS subject to publish to - * @param {string} message - JSON message to publish - * @param {string} correlationId - Correlation ID for tracing - * @param {boolean} [closeConnection=true] - Close connection after publish (set false for persistent connections) - */ -async function publishMessage(brokerUrlOrClient, subject, message, correlationId, closeConnection = true) { - let conn; - let shouldClose = false; - - if (brokerUrlOrClient instanceof NATSClient) { - conn = brokerUrlOrClient; - } else if (brokerUrlOrClient && typeof brokerUrlOrClient.publish === 'function') { - // Create a wrapper for direct connection (duck-typing check for NATS connection) - conn = { - async publish(subj, msg) { - await brokerUrlOrClient.publish(subj, msg); - }, - async close() { - await brokerUrlOrClient.close(); - } - }; - shouldClose = true; - } else { - // String URL - create new client - const client = new NATSClient(brokerUrlOrClient); - conn = client; - shouldClose = true; - } - - await conn.publish(subject, message, correlationId); - - // Only close if explicitly requested and it's a short-lived client - if (shouldClose && closeConnection && conn instanceof NATSClient) { - await conn.close(); - } -} - /** * Build message envelope from payloads and metadata - * @param {string} subject - NATS subject + * @param {string} subject - Subject/topic * @param {Array} payloads - Array of payload objects * @param {Object} options - Envelope metadata options * @returns {Object} Envelope object @@ -583,18 +400,21 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { } /** - * Send data via NATS with automatic transport selection + * Send data with automatic transport selection * * This function intelligently routes data delivery based on payload size. * If the serialized payload is smaller than size_threshold, it encodes the data as Base64 - * and publishes directly over NATS. Otherwise, it uploads the data to a fileserver - * and publishes only the download URL over NATS. + * into a "direct" payload. Otherwise, it uploads the data to a fileserver + * and creates a "link" payload with the URL. * - * @param {string} subject - NATS subject to publish the message to + * Transport publishing is the caller's responsibility. This function returns the + * envelope and its JSON string representation. + * + * @param {string} subject - Subject/topic to send the message to * @param {Array} data - List of [dataname, data, type] tuples to send * - type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * @param {Object} options - Optional configuration - * @param {string} [options.broker_url=DEFAULT_BROKER_URL] - URL of the NATS server + * @param {string} [options.broker_url=DEFAULT_BROKER_URL] - Broker URL (for envelope metadata) * @param {string} [options.fileserver_url=DEFAULT_FILESERVER_URL] - URL of the HTTP file server * @param {Function} [options.fileserver_upload_handler=plikOneshotUpload] - Function to handle fileserver uploads * @param {number} [options.size_threshold=DEFAULT_SIZE_THRESHOLD] - Threshold separating direct vs link transport @@ -605,8 +425,6 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * @param {string} [options.receiver_id=""] - UUID of the receiver (empty means broadcast) * @param {string} [options.reply_to=""] - Topic to reply to * @param {string} [options.reply_to_msg_id=""] - Message ID this message is replying to - * @param {boolean} [options.is_publish=true] - Whether to automatically publish the message - * @param {NATSClient|NATS.Connection} [options.nats_connection=null] - Pre-existing NATS connection * @param {string} [options.msg_id=crypto.randomUUID()] - Message ID * @param {string} [options.sender_id=crypto.randomUUID()] - Sender ID * @returns {Promise<[Object, string]>} Tuple of [env, env_json_str] @@ -615,8 +433,7 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * // Send a single payload * const [env, envJsonStr] = await smartsend( * "/test", - * [["dataname1", data1, "dictionary"]], - * { broker_url: "nats://localhost:4222" } + * [["dataname1", data1, "dictionary"]] * ); * * // Send multiple payloads @@ -625,17 +442,11 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * [ * ["dataname1", data1, "dictionary"], * ["dataname2", data2, "arrowtable"] - * ], - * { broker_url: "nats://localhost:4222" } + * ] * ); * - * // Send with pre-existing connection - * const client = await msghandler.NATSClient.connect("nats://localhost:4222"); - * const [env, envJsonStr] = await smartsend( - * "/test", - * [["data", myData, "text"]], - * { nats_connection: client } - * ); + * // Publish via your transport (NATS, MQTT, HTTP, etc.) + * // await myNatsClient.publish("/test", envJsonStr); */ async function smartsend(subject, data, options = {}) { const { @@ -650,8 +461,6 @@ async function smartsend(subject, data, options = {}) { receiver_id = '', reply_to = '', reply_to_msg_id = '', - is_publish = true, - nats_connection = null, msg_id = uuidv4(), sender_id = uuidv4() } = options; @@ -722,25 +531,18 @@ async function smartsend(subject, data, options = {}) { const env_json_str = JSON.stringify(env); - if (is_publish) { - if (nats_connection) { - await publishMessage(nats_connection, subject, env_json_str, correlation_id); - } else { - await publishMessage(broker_url, subject, env_json_str, correlation_id); - } - } - return [env, env_json_str]; } /** - * Receive and process NATS message + * Receive and process messages * - * This function processes incoming NATS messages, handling both direct transport + * This function processes incoming messages, handling both direct transport * (base64 decoded payloads) and link transport (URL-based payloads). * It deserializes the data based on the transport type and returns the result. * - * @param {Object} msg - NATS message object with payload property + * @param {string|Object} msg - Message payload. Accepts either a JSON string directly, + * or an object with a `data` or `payload` property containing the JSON string. * @param {Object} options - Optional configuration * @param {Function} [options.fileserver_download_handler=fetchWithBackoff] - Function to handle fileserver downloads * @param {number} [options.max_retries=5] - Maximum retry attempts for fetching URL @@ -749,13 +551,18 @@ async function smartsend(subject, data, options = {}) { * @returns {Promise} Envelope object with processed payloads * * @example - * // Receive and process message - * const env = await smartreceive(msg, { + * // Receive from JSON string directly + * const env = await smartreceive(jsonString, { * fileserver_download_handler: fetchWithBackoff, * max_retries: 5, * base_delay: 100, * max_delay: 5000 * }); + * + * // Receive from transport message object (e.g., NATS, MQTT) + * const env = await smartreceive(natsMsg, { + * fileserver_download_handler: fetchWithBackoff + * }); * // env.payloads is an Array of [dataname, data, type] arrays * for (const [dataname, data, type] of env.payloads) { * console.log(`${dataname}: ${data} (type: ${type})`); @@ -769,20 +576,20 @@ async function smartreceive(msg, options = {}) { max_delay = 5000 } = options; - // Debug: Log message object structure - logTrace('smartreceive', `smartreceive: msg object keys: ${Object.keys(msg).join(', ')}`); - logTrace('smartreceive', `smartreceive: msg.data type: ${typeof msg.data}, constructor: ${msg.data?.constructor?.name}`); - logTrace('smartreceive', `smartreceive: msg.payload type: ${typeof msg.payload}, constructor: ${msg.payload?.constructor?.name}`); - - // Parse the JSON envelope - // NATS.js v2.x uses msg.data instead of msg.payload + // Handle both raw JSON strings and transport message objects let payload; - if (msg.data !== undefined) { - payload = typeof msg.data === 'string' ? msg.data : Buffer.from(msg.data).toString('utf8'); - } else if (msg.payload !== undefined) { - payload = typeof msg.payload === 'string' ? msg.payload : Buffer.from(msg.payload).toString('utf8'); + if (typeof msg === 'string') { + payload = msg; + } else if (msg !== null && typeof msg === 'object') { + if (msg.data !== undefined) { + payload = typeof msg.data === 'string' ? msg.data : Buffer.from(msg.data).toString('utf8'); + } else if (msg.payload !== undefined) { + payload = typeof msg.payload === 'string' ? msg.payload : Buffer.from(msg.payload).toString('utf8'); + } else { + throw new Error('Message has neither data nor payload property'); + } } else { - throw new Error('Message has neither data nor payload property'); + throw new Error('Invalid message format: expected JSON string or message object'); } logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); @@ -866,61 +673,15 @@ async function smartreceive(msg, options = {}) { const msghandler = { /** - * NATS client class for connection management - * Supports both single-use and persistent connection modes - * - * @example - * // Single-use connection (closes after publish) - * const client = new msghandler.NATSClient("nats://localhost:4222"); - * await msghandler.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client }); - * await client.close(); - * - * // Persistent connection (keeps connection open) - * const client = new msghandler.NATSClient("nats://localhost:4222", true); - * await client.connect(); - * await msghandler.smartsend("/test1", [["msg", "Hello", "text"]], { nats_connection: client, is_publish: false }); - * await msghandler.publishMessage(client, "/test2", JSON.stringify({msg: "World"}), "trace-id"); - * // Connection remains open for more publishes - * await client.close(); - */ - NATSClient, - - /** - * Connection pool for managing multiple NATS connections - * Useful for applications with multiple concurrent publishers - * - * @example - * const pool = new msghandler.NATSConnectionPool("nats://localhost:4222", 10); - * const client = await pool.acquire(); - * await msghandler.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client }); - * pool.release(client); - * await pool.closeAll(); - */ - NATSConnectionPool, - - /** - * Send data via NATS with automatic transport selection + * Send data with automatic transport selection */ smartsend, /** - * Receive and process NATS message + * Receive and process messages */ smartreceive, - /** - * Publish message to NATS - * - * @example - * // Using a persistent connection - * const client = new msghandler.NATSClient("nats://localhost:4222", true); - * await client.connect(); - * await msghandler.publishMessage(client, "/subject", JSON.stringify({msg: "Hello"}), "trace-id", false); - * // Connection stays open for more publishes - * await client.close(); - */ - publishMessage, - /** * Upload data to plik server in one-shot mode */ @@ -939,4 +700,4 @@ const msghandler = { DEFAULT_FILESERVER_URL }; -module.exports = msghandler; \ No newline at end of file +module.exports = msghandler; diff --git a/src/natsbridge.py b/src/msghandler.py similarity index 78% rename from src/natsbridge.py rename to src/msghandler.py index 692a49f..f29f131 100644 --- a/src/natsbridge.py +++ b/src/msghandler.py @@ -3,8 +3,7 @@ msghandler - Cross-Platform Bi-Directional Data Bridge Python Desktop Implementation This module provides functionality for sending and receiving data across network boundaries -using NATS as the message bus, with support for both direct payload transport and -URL-based transport for larger payloads. +with support for both direct payload transport and URL-based transport for larger payloads. @package msghandler """ @@ -24,13 +23,6 @@ try: except ImportError: ARROW_AVAILABLE = False -try: - import nats - from nats.aio.client import Client as NATSClient - NATS_AVAILABLE = True -except ImportError: - NATS_AVAILABLE = False - # ---------------------------------------------- Constants ---------------------------------------------- # """ @@ -39,9 +31,9 @@ Default size threshold for switching from direct to link transport (0.5MB) DEFAULT_SIZE_THRESHOLD = 500_000 """ -Default NATS server URL +Default broker URL """ -DEFAULT_BROKER_URL = "nats://localhost:4222" +DEFAULT_BROKER_URL = "localhost:4222" """ Default HTTP file server URL for link transport @@ -305,56 +297,6 @@ async def fetch_with_backoff( raise Exception(f"Failed to fetch data after {max_retries} attempts") -# ---------------------------------------------- NATS Client ---------------------------------------------- # - -class NATSClient: - """NATS client wrapper for connection management.""" - - def __init__(self, url: str = DEFAULT_BROKER_URL): - """ - Create a new NATS client. - - Args: - url: NATS server URL - """ - self.url = url - self._client: NATSClient = None - - async def connect(self) -> NATSClient: - """ - Connect to NATS server. - - Returns: - NATS client instance - """ - if NATS_AVAILABLE: - self._client = nats.connect(self.url) - await self._client - else: - raise RuntimeError('nats-py not available') - return self._client - - async def publish(self, subject: str, message: str, correlation_id: str = "") -> None: - """ - Publish message to NATS subject. - - Args: - subject: NATS subject to publish to - message: Message to publish - correlation_id: Correlation ID for logging - """ - if self._client: - await self._client.publish(subject, message) - if correlation_id: - log_trace(correlation_id, f"Message published to {subject}") - - async def close(self) -> None: - """Close the NATS connection.""" - if self._client: - await self._client.drain() - await self._client.close() - - # ---------------------------------------------- Core Functions ---------------------------------------------- # def _build_envelope( @@ -366,7 +308,7 @@ def _build_envelope( Build message envelope from payloads and metadata. Args: - subject: NATS subject + subject: Subject/topic payloads: Array of payload objects options: Envelope metadata options @@ -430,41 +372,6 @@ def _build_payload( } -async def publish_message( - broker_url_or_client: Union[str, NATSClient, Any], - subject: str, - message: str, - correlation_id: str -) -> None: - """ - Publish message to NATS. - - Args: - broker_url_or_client: NATS URL, client, or connection - subject: NATS subject to publish to - message: JSON message to publish - correlation_id: Correlation ID for tracing - """ - if isinstance(broker_url_or_client, NATSClient): - client = broker_url_or_client - elif NATS_AVAILABLE and hasattr(broker_url_or_client, 'publish'): - # Direct NATS client connection - await broker_url_or_client.publish(subject, message) - log_trace(correlation_id, f"Message published to {subject}") - return - else: - # String URL - create new client - client = NATSClient(broker_url_or_client) - await client.connect() - - await client.publish(subject, message, correlation_id) - - if isinstance(broker_url_or_client, NATSClient): - await broker_url_or_client.close() - elif not (NATS_AVAILABLE and hasattr(broker_url_or_client, 'publish')): - await client.close() - - async def smartsend( subject: str, data: List[Tuple[str, Any, str]], @@ -479,26 +386,27 @@ async def smartsend( receiver_id: str = "", reply_to: str = "", reply_to_msg_id: str = "", - is_publish: bool = True, - nats_connection: Any = None, msg_id: str = None, sender_id: str = None ) -> Tuple[Dict, str]: """ - Send data via NATS with automatic transport selection. + Send data with automatic transport selection. This function intelligently routes data delivery based on payload size. If the serialized payload is smaller than size_threshold, it encodes the data as Base64 - and publishes directly over NATS. Otherwise, it uploads the data to a fileserver - and publishes only the download URL over NATS. + into a "direct" payload. Otherwise, it uploads the data to a fileserver + and creates a "link" payload with the URL. + + Transport publishing is the caller's responsibility. This function returns the + envelope and its JSON string representation. Args: - subject: NATS subject to publish the message to + subject: Subject/topic to send the message to data: List of (dataname, data, type) tuples to send - dataname: Name of the payload - data: The actual data to send - type: Payload type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" - broker_url: URL of the NATS server + broker_url: Broker URL (for envelope metadata) fileserver_url: URL of the HTTP file server for large payloads fileserver_upload_handler: Function to handle fileserver uploads (must return Dict with "status", "uploadid", "fileid", "url" keys) @@ -510,60 +418,24 @@ async def smartsend( receiver_id: UUID of the receiver (empty string means broadcast) reply_to: Topic to reply to (empty string if no reply expected) reply_to_msg_id: Message ID this message is replying to - is_publish: Whether to automatically publish the message to NATS - nats_connection: Pre-existing NATS connection (if provided, uses this connection instead of - creating a new one; saves connection establishment overhead) msg_id: Message ID (auto-generated UUID if not provided) sender_id: Sender ID (auto-generated UUID if not provided) Returns: Tuple of (env, env_json_str) where: - env: Dict containing all metadata and payloads - - env_json_str: JSON string for publishing to NATS + - env_json_str: JSON string for transport Example: >>> # Send a single payload (still wrapped in a list) >>> data = {"key": "value"} >>> env, env_json_str = await smartsend( ... "my.subject", - ... [("dataname1", data, "dictionary")], - ... broker_url="nats://localhost:4222" + ... [("dataname1", data, "dictionary")] ... ) >>> - >>> # Send multiple payloads with different types - >>> data1 = {"key1": "value1"} - >>> data2 = [1, 2, 3, 4, 5] - >>> env, env_json_str = await smartsend( - ... "my.subject", - ... [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")] - ... ) - >>> - >>> # Send a large array using fileserver upload - >>> data = list(range(10_000_000)) # ~80 MB - >>> env, env_json_str = await smartsend( - ... "large.data", - ... [("large_table", data, "arrowtable")] - ... ) - >>> - >>> # Send jsontable (JSON format for human-readable tabular data) - >>> users = [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] - >>> env, env_json_str = await smartsend( - ... "json.data", - ... [("users", users, "jsontable")] - ... ) - >>> - >>> # Mixed content (e.g., chat with text and image) - >>> env, env_json_str = await smartsend( - ... "chat.subject", - ... [ - ... ("message_text", "Hello!", "text"), - ... ("user_image", image_data, "image"), - ... ("audio_clip", audio_data, "audio") - ... ] - ... ) - >>> - >>> # Publish the JSON string directly using NATS request-reply pattern - >>> # reply = await nats.request(broker_url, subject, env_json_str, reply_to=reply_to_topic) + >>> # Publish the JSON string via your preferred transport + >>> # await my_nats_client.publish("my.subject", env_json_str) """ if correlation_id is None: correlation_id = str(uuid.uuid4()) @@ -619,12 +491,6 @@ async def smartsend( env_json_str = json.dumps(env) - if is_publish: - if nats_connection: - await publish_message(nats_connection, subject, env_json_str, correlation_id) - else: - await publish_message(broker_url, subject, env_json_str, correlation_id) - return env, env_json_str @@ -636,14 +502,15 @@ async def smartreceive( max_delay: int = 5000 ) -> Dict[str, Any]: """ - Receive and process NATS messages. + Receive and process messages. - This function processes incoming NATS messages, handling both direct transport + This function processes incoming messages, handling both direct transport (base64 decoded payloads) and link transport (URL-based payloads). It deserializes the data based on the transport type and returns the result. Args: - msg: NATS message to process + msg: Message to process. Accepts JSON string directly, or an object with + a `payload` or `data` property containing the JSON string. fileserver_download_handler: Function to handle downloading data from file server URLs max_retries: Maximum retry attempts for fetching URL base_delay: Initial delay for exponential backoff in ms @@ -653,10 +520,12 @@ async def smartreceive( Dict with envelope metadata and payloads field containing List[Tuple[str, Any, str]] Example: - >>> # Receive and process message - >>> env = await smartreceive(msg, fileserver_download_handler=fetch_with_backoff) + >>> # Receive from JSON string directly + >>> env = await smartreceive(json_string) + >>> + >>> # Receive from transport message object (e.g., NATS, MQTT) + >>> env = await smartreceive(nats_msg, fileserver_download_handler=fetch_with_backoff) >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] - >>> # Access payloads: for dataname, data, type_ in env["payloads"] >>> for dataname, data, type_ in env["payloads"]: >>> print(f"{dataname}: {data} (type: {type_})") """ @@ -664,13 +533,19 @@ async def smartreceive( if isinstance(msg, dict): # Already parsed env_json_obj = msg + elif isinstance(msg, str): + # Raw JSON string + env_json_obj = json.loads(msg) elif hasattr(msg, 'payload'): - # NATS message object + # Transport message object with payload property payload = msg.payload if isinstance(msg.payload, str) else msg.payload.decode('utf-8') env_json_obj = json.loads(payload) + elif hasattr(msg, 'data'): + # Transport message object with data property + payload = msg.data if isinstance(msg.data, str) else msg.data.decode('utf-8') + env_json_obj = json.loads(payload) else: - # Assume it's already a JSON string or dict - env_json_obj = json.loads(msg) if isinstance(msg, str) else msg + raise ValueError('Invalid message format: expected JSON string or message object') log_trace(env_json_obj['correlation_id'], "Processing received message") @@ -727,7 +602,7 @@ async def smartreceive( class msghandler: """ - Cross-platform NATS bridge implementation. + Cross-platform message bridge implementation. This class provides a convenient interface for msghandler functionality, encapsulating the main functions and providing a class-based API. @@ -742,7 +617,7 @@ class msghandler: Initialize msghandler. Args: - broker_url: NATS server URL (defaults to DEFAULT_BROKER_URL) + broker_url: Broker URL (defaults to DEFAULT_BROKER_URL) fileserver_url: HTTP file server URL (defaults to DEFAULT_FILESERVER_URL) """ self.broker_url = broker_url or self.DEFAULT_BROKER_URL @@ -755,10 +630,10 @@ class msghandler: **kwargs ) -> Tuple[Dict, str]: """ - Send data via NATS. + Send data. Args: - subject: NATS subject to publish to + subject: Subject/topic to send to data: List of (dataname, data, type) tuples **kwargs: Additional options passed to smartsend @@ -775,10 +650,10 @@ class msghandler: **kwargs ) -> Dict[str, Any]: """ - Receive and process NATS message. + Receive and process message. Args: - msg: NATS message to process + msg: Message to process **kwargs: Additional options passed to smartreceive Returns: @@ -797,7 +672,7 @@ def send( Convenience function for sending data. Args: - subject: NATS subject to publish to + subject: Subject/topic to send to data: List of (dataname, data, type) tuples **kwargs: Additional options @@ -815,7 +690,7 @@ def receive( Convenience function for receiving messages. Args: - msg: NATS message to process + msg: Message to process **kwargs: Additional options Returns: @@ -835,9 +710,7 @@ __all__ = [ 'DEFAULT_SIZE_THRESHOLD', 'DEFAULT_BROKER_URL', 'DEFAULT_FILESERVER_URL', - 'NATSClient', '_serialize_data', '_deserialize_data', - 'log_trace', - 'publish_message' -] \ No newline at end of file + 'log_trace' +] diff --git a/src/natsbridge.rs b/src/msghandler.rs similarity index 97% rename from src/natsbridge.rs rename to src/msghandler.rs index d84a312..eb0dec4 100644 --- a/src/natsbridge.rs +++ b/src/msghandler.rs @@ -1,6 +1,6 @@ // msghandler Rust Module -// Cross-platform bi-directional data bridge for NATS communication -// Implements smartsend and smartreceive for NATS communication +// Cross-platform bi-directional data bridge +// Implements smartsend and smartreceive for message transport // with support for both direct payload transport and URL-based transport // for larger payloads using the Claim-Check pattern. // @@ -38,8 +38,8 @@ use uuid::Uuid; /// Default size threshold (0.5MB) for switching from direct to link transport pub const DEFAULT_SIZE_THRESHOLD: usize = 500_000; -/// Default NATS server URL -pub const DEFAULT_BROKER_URL: &str = "nats://localhost:4222"; +/// Default broker URL +pub const DEFAULT_BROKER_URL: &str = "localhost:4222"; /// Default HTTP file server URL for link transport pub const DEFAULT_FILESERVER_URL: &str = "http://localhost:8080"; @@ -68,8 +68,8 @@ pub enum msghandlerError { DownloadFailed { url: String, retries: u32 }, /// Unknown transport type UnknownTransport(String), - /// NATS connection failed - NatConnectionFailed(String), + /// Connection failed + ConnectionFailed(String), /// Payload deserialization error DeserializationError(String), /// HTTP request error @@ -95,7 +95,7 @@ impl fmt::Display for msghandlerError { write!(f, "Failed to fetch {} after {} attempts", url, retries) } msghandlerError::UnknownTransport(t) => write!(f, "Unknown transport type: {}", t), - msghandlerError::NatConnectionFailed(msg) => write!(f, "NATS connection failed: {}", msg), + msghandlerError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg), msghandlerError::DeserializationError(msg) => { write!(f, "Deserialization error: {}", msg) } @@ -172,7 +172,7 @@ impl Payload { // Message Payload Structure (wire format) // ============================================================================ -/// Represents a single payload within a NATS message envelope. +/// Represents a single payload within a message envelope. /// Supports both direct transport (base64-encoded data) and link transport (URL-based). #[derive(Serialize, Deserialize, Clone, Debug, Default)] pub struct MsgPayloadV1 { @@ -257,7 +257,7 @@ impl MsgPayloadV1 { // Message Envelope Structure (wire format) // ============================================================================ -/// Represents a complete NATS message envelope containing multiple payloads +/// Represents a complete message envelope containing multiple payloads /// with metadata for routing, tracing, and message context. #[derive(Serialize, Deserialize, Clone, Debug, Default)] pub struct MsgEnvelopeV1 { @@ -268,7 +268,7 @@ pub struct MsgEnvelopeV1 { /// Message publication timestamp (ISO 8601 UTC) pub timestamp: String, - /// NATS subject/topic to publish the message to + /// Subject/topic to send the message to pub send_to: String, /// Purpose of the message: "ACK", "NACK", "updateStatus", "shutdown", /// "chat", "command", "event" @@ -286,7 +286,7 @@ pub struct MsgEnvelopeV1 { pub reply_to: String, /// Message ID this message is replying to pub reply_to_msg_id: String, - /// NATS broker URL + /// Broker URL pub broker_url: String, /// Optional message-level metadata @@ -317,7 +317,7 @@ impl MsgEnvelopeV1 { } } - /// Convert the envelope to a JSON string for NATS publishing + /// Convert the envelope to a JSON string for transport pub fn to_json(&self) -> Result { serde_json::to_string(self).map_err(|e| msghandlerError::JsonError(e.to_string())) } @@ -329,7 +329,7 @@ impl MsgEnvelopeV1 { /// Options for the `smartsend` function pub struct SmartsendOptions { - /// NATS server URL + /// Broker URL pub broker_url: String, /// HTTP file server URL for large payloads pub fileserver_url: String, @@ -701,7 +701,7 @@ pub fn log_trace(correlation_id: &str, message: &str) { // Public API: smartsend // ============================================================================ -/// Send data via NATS with automatic transport selection. +/// Send data with automatic transport selection. /// /// This function intelligently routes data delivery based on payload size. /// If the serialized payload is smaller than `size_threshold`, it encodes the @@ -711,11 +711,11 @@ pub fn log_trace(correlation_id: &str, message: &str) { /// Each payload in the list can have a different type, enabling mixed-content /// messages (e.g., chat with text, images, audio). /// -/// NATS publishing is the caller's responsibility. This function returns the +/// Transport publishing is the caller's responsibility. This function returns the /// envelope and its JSON string representation. /// /// # Arguments -/// - `subject`: NATS subject to publish the message to +/// - `subject`: Subject/topic to send the message to /// - `data`: Slice of (dataname, payload, payload_type) tuples /// - `options`: Configuration options /// @@ -736,8 +736,7 @@ pub fn log_trace(correlation_id: &str, message: &str) { /// &SmartsendOptions::default(), /// ).await?; /// -/// // Caller publishes to NATS -/// // conn.publish("/agent/wine/api/v1/prompt", &json_str)?; +/// // Caller publishes via their preferred transport /// # Ok(()) /// # } /// ``` @@ -793,7 +792,7 @@ pub async fn smartsend( )); if payload_size < options.size_threshold { - // Direct transport: Base64 encode and include in NATS message + // Direct transport: Base64 encode and include in message envelope let payload_b64 = BASE64.encode(&payload_bytes); log_trace(&correlation_id, &format!( "Using direct transport for {} bytes", payload_size @@ -807,7 +806,7 @@ pub async fn smartsend( ); payloads.push(msg_payload); } else { - // Link transport: Upload to file server, include URL in NATS message + // Link transport: Upload to file server, include URL in message envelope log_trace(&correlation_id, "Using link transport, uploading to fileserver"); let upload_result = upload_handler @@ -878,15 +877,15 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms // Public API: smartreceive // ============================================================================ -/// Receive and process messages from NATS. +/// Receive and process messages. /// -/// This function processes incoming NATS messages, handling both direct transport +/// This function processes incoming messages, handling both direct transport /// (base64 decoded payloads) and link transport (URL-based payloads). /// It deserializes the data based on the payload type and returns the envelope /// with deserialized payloads. /// /// # Arguments -/// - `msg_json_str`: JSON string from NATS message payload +/// - `msg_json_str`: JSON string from the message payload /// - `options`: Configuration options /// /// # Returns @@ -902,7 +901,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// "timestamp":"2026-01-01T00:00:00Z","send_to":"/test", /// "msg_purpose":"chat","sender_name":"test","sender_id":"sender-uuid", /// "receiver_name":"","receiver_id":"","reply_to":"","reply_to_msg_id":"", -/// "broker_url":"nats://localhost:4222","payloads":[{ +/// "broker_url":"localhost:4222","payloads":[{ /// "id":"payload-uuid","dataname":"msg","payload_type":"text", /// "transport":"direct","encoding":"base64","size":5, /// "data":"SGVsbG8=","metadata":{"payload_bytes":5} diff --git a/src/natsbridge_mpy.py b/src/msghandler_mpy.py similarity index 82% rename from src/natsbridge_mpy.py rename to src/msghandler_mpy.py index 2296530..e6ec6ab 100644 --- a/src/natsbridge_mpy.py +++ b/src/msghandler_mpy.py @@ -3,8 +3,7 @@ msghandler - Cross-Platform Bi-Directional Data Bridge MicroPython Implementation This module provides functionality for sending and receiving data across network boundaries -using NATS as the message bus, with support for both direct payload transport and -URL-based transport for larger payloads. +with support for both direct payload transport and URL-based transport for larger payloads. Note: MicroPython has significant constraints compared to desktop implementations: - Limited memory (~256KB - 1MB) @@ -29,9 +28,9 @@ Default size threshold for switching from direct to link transport (100KB for Mi DEFAULT_SIZE_THRESHOLD = 100000 """ -Default NATS server URL +Default broker URL """ -DEFAULT_BROKER_URL = "nats://localhost:4222" +DEFAULT_BROKER_URL = "localhost:4222" """ Default HTTP file server URL for link transport @@ -190,64 +189,6 @@ def _sync_fileserver_download(url, max_retries, base_delay, max_delay, correlati "Use direct transport only for memory-constrained devices.") -# ---------------------------------------------- NATS Client ---------------------------------------------- # - -class NATSClient: - """ - NATS client wrapper for MicroPython. - - Note: - This is a simplified implementation for MicroPython. - Full NATS client implementation would require additional network stack support. - """ - - def __init__(self, url=DEFAULT_BROKER_URL): - """ - Initialize NATS client. - - Args: - url: NATS server URL - """ - self.url = url - self._connected = False - - def connect(self): - """ - Connect to NATS server. - - Note: - This is a placeholder implementation. - Actual NATS client would require network stack support. - - Returns: - True if connected, False otherwise - """ - # Placeholder - actual implementation would connect to NATS server - self._connected = True - return self._connected - - def publish(self, subject, message): - """ - Publish message to NATS subject. - - Note: - This is a placeholder implementation. - Actual NATS client would require network stack support. - - Args: - subject: NATS subject to publish to - message: Message to publish - """ - if not self._connected: - raise RuntimeError("Not connected to NATS server") - # Placeholder - actual implementation would publish to NATS - print(f"[NATS] Publish to {subject}: {message[:50]}...") - - def close(self): - """Close the NATS connection.""" - self._connected = False - - # ---------------------------------------------- Core Functions ---------------------------------------------- # def _build_envelope(subject, payloads, options): @@ -255,7 +196,7 @@ def _build_envelope(subject, payloads, options): Build message envelope from payloads and metadata. Args: - subject: NATS subject + subject: Subject/topic payloads: Array of payload objects options: Envelope metadata options @@ -308,44 +249,43 @@ def _build_payload(dataname, payload_type, payload_bytes, transport, data): def _publish(subject, message, correlation_id): """ - Publish message to NATS. + Publish message via transport. Note: This is a simplified implementation for MicroPython. Args: - subject: NATS subject to publish to + subject: Subject to publish to message: JSON message to publish correlation_id: Correlation ID for logging """ log_trace(correlation_id, f"Publishing to {subject}") - # Placeholder - actual implementation would use NATSClient - # client = NATSClient() - # client.connect() - # client.publish(subject, message) - # client.close() + # Placeholder - actual implementation would publish via preferred transport def smartsend(subject, data, **kwargs): """ - Send data via NATS with automatic transport selection. + Send data with automatic transport selection. This function intelligently routes data delivery based on payload size. If the serialized payload is smaller than size_threshold, it encodes the data as Base64 - and publishes directly over NATS. Otherwise, it uploads the data to a fileserver - and publishes only the download URL over NATS. + into a "direct" payload. Otherwise, it uploads the data to a fileserver + and creates a "link" payload with the URL. + + Transport publishing is the caller's responsibility. This function returns the + envelope and its JSON string representation. Note: MicroPython has memory constraints, so the default size_threshold is lower (100KB). Table type is not supported due to memory constraints. Args: - subject: NATS subject to publish the message to + subject: Subject/topic to send the message to data: List of (dataname, data, type) tuples to send - dataname: Name of the payload - data: The actual data to send - type: Payload type: "text", "dictionary", "image", "audio", "video", "binary" - broker_url: NATS server URL (default: DEFAULT_BROKER_URL) + broker_url: Broker URL (for envelope metadata, default: DEFAULT_BROKER_URL) fileserver_url: HTTP file server URL (default: DEFAULT_FILESERVER_URL) fileserver_upload_handler: Function to handle fileserver uploads (default: _sync_fileserver_upload) size_threshold: Threshold in bytes separating direct vs link transport (default: 100000) @@ -356,36 +296,23 @@ def smartsend(subject, data, **kwargs): receiver_id: UUID of the receiver (empty means broadcast) reply_to: Topic to reply to (empty if no reply expected) reply_to_msg_id: Message ID this message is replying to - is_publish: Whether to automatically publish the message (default: True) msg_id: Message ID (auto-generated if not provided) sender_id: Sender ID (auto-generated if not provided) Returns: Tuple of (env, env_json_str) where: - env: Dict containing all metadata and payloads - - env_json_str: JSON string for publishing to NATS + - env_json_str: JSON string for transport Example: >>> # Send text payload - >>> env, env_json_str = msghandler.smartsend( + >>> env, env_json_str = smartsend( ... "/chat", - ... [("message", "Hello!", "text")], - ... broker_url="nats://localhost:4222" + ... [("message", "Hello!", "text")] ... ) >>> - >>> # Send dictionary payload - >>> env, env_json_str = msghandler.smartsend( - ... "/config", - ... [("config", {"key": "value"}, "dictionary")], - ... broker_url="nats://localhost:4222" - ... ) - >>> - >>> # Send binary payload (image, audio, video) - >>> env, env_json_str = msghandler.smartsend( - ... "/media", - ... [("image", image_bytes, "image")], - ... broker_url="nats://localhost:4222" - ... ) + >>> # Publish via your transport + >>> # my_transport.publish("/chat", env_json_str) """ # Extract options with defaults correlation_id = kwargs.get('correlation_id', _generate_uuid()) @@ -465,9 +392,9 @@ def smartsend(subject, data, **kwargs): def smartreceive(msg, **kwargs): """ - Receive and process NATS message. + Receive and process messages. - This function processes incoming NATS messages, handling both direct transport + This function processes incoming messages, handling both direct transport (base64 decoded payloads) and link transport (URL-based payloads). It deserializes the data based on the transport type and returns the result. @@ -476,7 +403,7 @@ def smartreceive(msg, **kwargs): Table type is not supported due to memory constraints. Args: - msg: NATS message to process (can be string, dict, or object with 'payload' attribute) + msg: Message to process (can be JSON string, dict, or object with 'payload'/'data' attribute) fileserver_download_handler: Function to handle downloading data from file server URLs max_retries: Maximum retry attempts (default: 3) base_delay: Initial delay in ms (default: 100) @@ -486,8 +413,11 @@ def smartreceive(msg, **kwargs): Dict with envelope metadata and payloads field containing List[Tuple[str, Any, str]] Example: - >>> # Receive and process message - >>> env = msghandler.smartreceive(msg, fileserver_download_handler=_sync_fileserver_download) + >>> # Receive from JSON string + >>> env = smartreceive(json_string) + >>> + >>> # Receive from transport message object + >>> env = smartreceive(transport_msg, fileserver_download_handler=_sync_fileserver_download) >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] >>> for dataname, data, type_ in env["payloads"]: ... print(f"{dataname}: {data} (type: {type_})") @@ -496,13 +426,19 @@ def smartreceive(msg, **kwargs): if isinstance(msg, dict): # Already parsed env_json_obj = msg + elif isinstance(msg, str): + # Raw JSON string + env_json_obj = json.loads(msg) elif hasattr(msg, 'payload'): # Object with payload attribute payload = msg.payload if isinstance(msg.payload, str) else msg.payload.decode('utf-8') env_json_obj = json.loads(payload) + elif hasattr(msg, 'data'): + # Object with data attribute + payload = msg.data if isinstance(msg.data, str) else msg.data.decode('utf-8') + env_json_obj = json.loads(payload) else: - # Assume it's already a JSON string or dict - env_json_obj = json.loads(msg) if isinstance(msg, str) else msg + raise ValueError('Invalid message format: expected JSON string or message object') correlation_id = env_json_obj['correlation_id'] log_trace(correlation_id, "Processing received message") @@ -565,7 +501,7 @@ def smartreceive(msg, **kwargs): class msghandler: """ - MicroPython NATS bridge implementation. + MicroPython message bridge implementation. This class provides a convenient interface for msghandler functionality, encapsulating the main functions and providing a class-based API. @@ -588,7 +524,7 @@ class msghandler: Initialize msghandler. Args: - broker_url: NATS server URL (defaults to DEFAULT_BROKER_URL) + broker_url: Broker URL (defaults to DEFAULT_BROKER_URL) fileserver_url: HTTP file server URL (defaults to DEFAULT_FILESERVER_URL) """ self.broker_url = broker_url or self.DEFAULT_BROKER_URL @@ -596,10 +532,10 @@ class msghandler: def smartsend(self, subject, data, **kwargs): """ - Send data via NATS. + Send data. Args: - subject: NATS subject to publish to + subject: Subject/topic to send to data: List of (dataname, data, type) tuples **kwargs: Additional options passed to smartsend @@ -612,10 +548,10 @@ class msghandler: def smartreceive(self, msg, **kwargs): """ - Receive and process NATS message. + Receive and process message. Args: - msg: NATS message to process + msg: Message to process **kwargs: Additional options passed to smartreceive Returns: @@ -630,7 +566,7 @@ def send(subject, data, **kwargs): Convenience function for sending data. Args: - subject: NATS subject to publish to + subject: Subject/topic to send to data: List of (dataname, data, type) tuples **kwargs: Additional options @@ -645,7 +581,7 @@ def receive(msg, **kwargs): Convenience function for receiving messages. Args: - msg: NATS message to process + msg: Message to process **kwargs: Additional options Returns: @@ -664,10 +600,9 @@ __all__ = [ 'DEFAULT_BROKER_URL', 'DEFAULT_FILESERVER_URL', 'MAX_PAYLOAD_SIZE', - 'NATSClient', '_serialize_data', '_deserialize_data', 'log_trace', '_sync_fileserver_upload', '_sync_fileserver_download' -] \ No newline at end of file +] From d0910ccc3f5cff6e7b826e46a4eb60f88ffdd2ae Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 15 May 2026 13:25:48 +0700 Subject: [PATCH 02/14] update docs --- docs/architecture.md | 127 +++++++++++++++++++++--------------------- docs/requirements.md | 75 +++++++++++++------------ docs/specification.md | 114 ++++++++++++++++++------------------- docs/walkthrough.md | 95 ++++++++++++++++--------------- 4 files changed, 206 insertions(+), 205 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 470c2a6..458677f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ ## 1. Executive Summary -This document defines the **blueprint** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using NATS as the message bus. +This document defines the **blueprint** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. This architecture document serves as the single source of truth for: - **System Structure**: How components fit together and interact @@ -47,26 +47,25 @@ This architecture document serves as the single source of truth for: ```mermaid flowchart TD - subgraph "External Systems" - NATS_Server[NATS Server] + subgraph "External Systems" + Message_Broker[Message Broker
NATS/MQTT/WebSocket/Custom] File_Server[HTTP File Server
Plik/AWS S3/Custom] end - subgraph "Client Applications" - Julia_App[Julia Application] - JS_App[JavaScript Application
Node.js/Browser] - Python_App[Python Application
Desktop] - Dart_App[Dart Application
Desktop/Flutter/Web] - Rust_App[Rust Application
Server/Desktop] - MicroPython_App[MicroPython Device] + Julia_App[Julia Application] + JS_App[JavaScript Application
Node.js/Browser] + Python_App[Python Application
Desktop] + Dart_App[Dart Application
Desktop/Flutter/Web] + Rust_App[Rust Application
Server/Desktop] + MicroPython_App[MicroPython Device] end - Julia_App -->|NATS| NATS_Server - JS_App -->|NATS| NATS_Server - Python_App -->|NATS| NATS_Server - Dart_App -->|NATS| NATS_Server - Rust_App -->|NATS| NATS_Server - MicroPython_App -->|NATS| NATS_Server + Julia_App -->|Transport| Message_Broker + JS_App -->|Transport| Message_Broker + Python_App -->|Transport| Message_Broker + Dart_App -->|Transport| Message_Broker + Rust_App -->|Transport| Message_Broker + MicroPython_App -->|Transport| Message_Broker Julia_App -->|HTTP| File_Server JS_App -->|HTTP| File_Server @@ -75,7 +74,7 @@ flowchart TD Rust_App -->|HTTP| File_Server MicroPython_App -->|HTTP| File_Server - style NATS_Server fill:#fff3e0,stroke:#f57c00 + style Message_Broker fill:#fff3e0,stroke:#f57c00 style File_Server fill:#f3e5f5,stroke:#9c27b4 style Julia_App fill:#e8f5e9,stroke:#4caf50 style JS_App fill:#e3f2fd,stroke:#2196f3 @@ -98,14 +97,14 @@ flowchart TD MicroPython_Module[MicroPython msghandler Module] end - Julia_Module --> NATS_Client - JS_Module --> NATS_Client - Python_Module --> NATS_Client - Dart_Module --> NATS_Client - Rust_Module --> NATS_Client - MicroPython_Module --> NATS_Client + Julia_Module --> Transport_Client + JS_Module --> Transport_Client + Python_Module --> Transport_Client + Dart_Module --> Transport_Client + Rust_Module --> Transport_Client + MicroPython_Module --> Transport_Client - NATS_Client --> NATS_Broker + Transport_Client --> Message_Broker Julia_Module --> File_Client JS_Module --> File_Client @@ -122,7 +121,7 @@ flowchart TD style Dart_Module fill:#fff0f6,stroke:#e91e63 style Rust_Module fill:#dea584,stroke:#e65100 style MicroPython_Module fill:#fce4ec,stroke:#e91e63 - style NATS_Broker fill:#fff3e0,stroke:#f57c00 + style Message_Broker fill:#fff3e0,stroke:#f57c00 style File_Server fill:#f3e5f5,stroke:#9c27b4 ``` @@ -174,8 +173,8 @@ flowchart TD | Component | Purpose | Platform Support | |-----------|---------|------------------| -| **smartsend** | Send data via NATS with automatic transport selection, returns (envelope, json_string) for caller to publish | All | -| **smartreceive** | Receive and process NATS messages from JSON string | All | +| **smartsend** | Send data with automatic transport selection, returns (envelope, json_string) for caller to publish via transport | All | +| **smartreceive** | Receive and process messages from JSON string | All | | **_serialize_data** | Serialize data according to payload type | All | | **_deserialize_data** | Deserialize bytes to native data types | All | | **envelope_to_json** | Convert msg_envelope_v1 struct to JSON string | All | @@ -224,7 +223,7 @@ struct msg_envelope_v1 msg_id::String # UUID v4 for this message timestamp::String # ISO 8601 UTC timestamp - send_to::String # NATS subject to publish to + send_to::String # Topic/subject to publish to msg_purpose::String # ACK, NACK, updateStatus, shutdown, chat sender_name::String # Sender application name sender_id::String # UUID v4 of sender @@ -233,7 +232,7 @@ struct msg_envelope_v1 reply_to::String # Topic for reply messages reply_to_msg_id::String # Message ID being replied to - broker_url::String # NATS broker URL + broker_url::String # Broker URL for the transport layer metadata::Dict{String, Any} # Message-level metadata payloads::Vector{msg_payload_v1} # List of payloads @@ -436,13 +435,13 @@ JavaScript uses async/await for non-blocking I/O: #### Node.js Implementation (msghandler_ssr.js) -- **TCP NATS connections**: Uses `nats://` or `tls://` URLs +- **Transport connections**: Uses broker URLs (e.g., `nats://`, `mqtt://`, `ws://`) - **Apache Arrow IPC**: Full support via `apache-arrow` - **Buffer for binary data**: Native Node.js Buffer handling #### Browser Implementation (msghandler_csr.js) -- **WebSocket NATS connections**: Uses `ws://` or `wss://` URLs via `nats.ws` +- **WebSocket connections**: Uses `ws://` or `wss://` URLs (transport-agnostic) - **No Apache Arrow**: Uses `jsontable` for tabular data only - **Uint8Array for binary data**: Browser-compatible binary handling - **Web Crypto API**: UUID generation via `crypto.getRandomValues()` @@ -474,7 +473,7 @@ Dart uses classes for stateful operations with async/await: - **Async/await**: I/O operations - **dart-arrow**: Arrow IPC support (Desktop/Flutter only) - **HTTP package**: HTTP file server communication -- **nats package**: NATS client with WebSocket support (Dart Web) +- **Transport package**: Transport client with WebSocket support (Dart Web) ```dart class msghandler { @@ -484,7 +483,7 @@ class msghandler { final String fileserverUrl; msghandler({ - this.brokerUrl = 'nats://localhost:4222', + this.brokerUrl = DEFAULT_BROKER_URL, this.fileserverUrl = 'http://localhost:8080', }); } @@ -492,19 +491,19 @@ class msghandler { #### Dart Desktop (Dart SDK) -- **TCP NATS connections**: Uses `nats://` or `tls://` URLs +- **Transport connections**: Uses broker URLs (e.g., `nats://`, `mqtt://`) - **Apache Arrow IPC**: Full support via `dart-arrow` - **Uint8List for binary data**: Native Dart binary handling #### Dart Flutter (Dart SDK) -- **TCP NATS connections**: Uses `nats://` or `tls://` URLs +- **Transport connections**: Uses broker URLs (e.g., `nats://`, `mqtt://`) - **Apache Arrow IPC**: Full support via `dart-arrow` - **Uint8List for binary data**: Native Dart binary handling #### Dart Web (Dart SDK) -- **WebSocket NATS connections**: Uses `ws://` or `wss://` URLs via `nats` package +- **WebSocket connections**: Uses `ws://` or `wss://` URLs (transport-agnostic) - **No Apache Arrow**: Uses `jsontable` for tabular data only - **Uint8List for binary data**: Browser-compatible binary handling - **Fetch API**: HTTP file server communication via `http` package @@ -516,7 +515,7 @@ Browser JavaScript has specific constraints due to security and compatibility: - **Async/await**: Native async/await support - **No Apache Arrow**: Arrow IPC not available in browsers - **JSON table only**: Use "jsontable" for tabular data -- **WebSocket NATS**: Uses nats.ws for browser-compatible NATS connections +- **WebSocket transport**: Uses transport client for browser-compatible connections - **Fetch API**: HTTP file server communication via fetch ### MicroPython Architecture @@ -540,7 +539,7 @@ Rust leverages compile-time type safety and async runtimes: - **Type-safe payloads**: Rust enum discriminates between `Text`, `Dictionary`, `ArrowTable`, `Binary`, etc. - **serde serialization**: Automatic JSON deserialization via `#[derive(Serialize, Deserialize)]` -- **tokio runtime**: Efficient async I/O for NATS connections and HTTP file server operations +- **tokio runtime**: Efficient async I/O for transport connections and HTTP file server operations - **arrow2 integration**: Native Arrow IPC deserialization without intermediate format conversion - **reqwest**: High-performance HTTP client with built-in TLS and connection pooling - **Zero-copy patterns**: `Vec` passed directly to avoid unnecessary memory copies @@ -572,8 +571,8 @@ pub struct SmartsendOptions { // ... other fields } -// NATS client with tokio integration -let conn = nats::connect("nats://localhost:4222").await?; +// Transport client with tokio integration +let conn = transport_client::connect(DEFAULT_BROKER_URL).await?; // Subscribe and process messages let mut sub = conn.subscribe("/agent/wine/api/v1/analyze")?; @@ -599,7 +598,7 @@ for msg in sub.messages() { | Component | Scaling Strategy | |-----------|------------------| -| **NATS Server** | Cluster deployment with multiple nodes | +| **Message Broker** | Cluster deployment with multiple nodes | | **File Server** | Load balancer + multiple instances | | **Client Applications** | Deploy multiple instances behind load balancer | @@ -607,7 +606,7 @@ for msg in sub.messages() { | Component | Scaling Strategy | |-----------|------------------| -| **NATS Server** | Increase memory, CPU, disk I/O | +| **Message Broker** | Increase memory, CPU, disk I/O | | **File Server** | Increase memory, CPU, disk capacity | | **Client Applications** | Increase heap size (Python/JS) | @@ -617,7 +616,7 @@ for msg in sub.messages() { |--------|--------|-------| | Message serialization overhead | <50ms | For 10KB payload | | Message deserialization overhead | <50ms | For 10KB payload | -| NATS connection establishment | <100ms | Connection pool recommended | +| Transport connection establishment | <100ms | Connection pool recommended | | File upload latency | <1s | For 0.5MB file | | File download latency | <1s | For 0.5MB file | @@ -625,16 +624,16 @@ for msg in sub.messages() { ## Failure Modes and Recovery -### NATS Connection Failure +### Transport Connection Failure -**Scenario**: NATS server unavailable +**Scenario**: Message broker unavailable **Handler**: -- Connection auto-reconnect via TCP-level reconnection +- Connection auto-reconnect via transport-level reconnection - Retry with exponential backoff for publish operations **Recovery**: -- NATS client automatically attempts reconnection +- Transport client automatically attempts reconnection - Application can check connection status before publishing ### File Server Unavailable @@ -697,7 +696,7 @@ for msg in sub.messages() { **Rationale**: - Simplifies JSON serialization (all data is string-compatible) -- Increases payload size by ~33%, but NATS can handle this +- Increases payload size by ~33%, but transport can handle this - Alternative would be binary payload support (more complex) ### Decision 3: Multiple Platform Implementations @@ -730,7 +729,7 @@ for msg in sub.messages() { | Component | Minimum | Notes | |-----------|---------|-------| -| NATS Server | 1 instance | Single node for development | +| Message Broker | 1 instance | Single node for development | | File Server | 1 instance | HTTP server for large payloads | | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) | | Client Memory | 256KB | MicroPython devices | @@ -739,7 +738,7 @@ for msg in sub.messages() { | Variable | Default | Description | |----------|---------|-------------| -| `NATS_URL` | `nats://localhost:4222` | NATS server URL | +| `BROKER_URL` | `ws://localhost:4222` | Message broker URL | | `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL | | `SIZE_THRESHOLD` | `500000` | Size threshold in bytes (0.5MB) | @@ -748,15 +747,15 @@ for msg in sub.messages() { ```mermaid flowchart TD subgraph "Docker Network" - NATS_Container[NATS Server] + Broker_Container[Message Broker] FileServer_Container[Plik File Server] App_Container[Application Container] end - App_Container -->|NATS| NATS_Container + App_Container -->|Transport| Broker_Container App_Container -->|HTTP| FileServer_Container - style NATS_Container fill:#fff3e0,stroke:#f57c00 + style Broker_Container fill:#fff3e0,stroke:#f57c00 style FileServer_Container fill:#f3e5f5,stroke:#9c27b4 style App_Container fill:#e3f2fd,stroke:#2196f3 ``` @@ -775,11 +774,12 @@ flowchart TD ### Transport Security -**Mechanism**: TLS support for NATS connections +**Mechanism**: TLS support for transport connections **Implementation**: - Use `nats://` URL for plain text - Use `tls://` URL for TLS-encrypted connections +- Use `ws://` or `wss://` for WebSocket connections ### File Server Security @@ -835,6 +835,11 @@ flowchart TD | Date | Version | Changes | |------|---------|---------| +| 2026-05-15 | 1.5.0 | Made transport layer agnostic | All sections | +| - | - | Removed all NATS-specific references from architecture docs | All sections | +| - | - | Updated diagrams to use generic "Message Broker" instead of "NATS Server" | All sections | +| - | - | Updated code examples to use transport-agnostic patterns | All sections | +| - | - | Removed NATS client packages from external dependencies | All sections | | 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartreceive` deserialization changes | All sections | | - | - | `smartreceive` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | | - | - | Added `plik_upload_file` convenience function to component table | specification.md:13 | @@ -849,6 +854,7 @@ flowchart TD | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | | - | - | Removed publish_message component (commented out in source) | | - | - | Removed NATSClient and NATSConnectionPool classes (not in ground truth) | +| - | - | Updated smartsend to return JSON for caller to publish via transport | | - | - | Updated component diagram to match actual module structure | | - | - | Updated data flow to show smartsend returns JSON for caller to publish | | - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | @@ -856,6 +862,7 @@ flowchart TD | - | - | Added NATSClient with keepAlive support | | - | - | Added NATSConnectionPool for connection reuse | | - | - | Added publishMessage function with closeConnection option | +| (Historical - pre-transport-agnostic refactor) | | | | 2026-03-13 | 1.0.0 | Initial architecture documentation | --- @@ -880,7 +887,7 @@ flowchart TD |------|----------|----------|---------------------------|-------------------| | [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only, WebSocket NATS | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe, file upload helpers | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | @@ -889,24 +896,18 @@ flowchart TD ### 16.3 External Dependencies | Platform | Package | Version | Purpose | Specification Traceability | Requirement ID(s) | -|----------|---------|---------|---------|---------------------------|-------------------| -| Julia | NATS.jl | Latest | NATS client | specification.md:11 | FR-013, FR-014, NFR-201 | +|----------|---------|---------|---------|--------------------------|-------------------| | Julia | JSON.jl | Latest | JSON serialization | specification.md:11 | FR-012, NFR-101, NFR-102 | | Julia | Arrow.jl | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | | Julia | HTTP.jl | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | | Julia | UUIDs.jl | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 | -| Node.js | nats | Latest | NATS client (TCP) | specification.md:11 | FR-013, FR-014 | | Node.js | node-fetch | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | -| Browser | nats.ws | Latest | NATS client (WebSocket) | specification.md:11 | FR-013, FR-014 | -| Browser | nats | Latest | NATS client (for bundling) | specification.md:11 | FR-013, FR-014 | -| Python | nats-py | Latest | NATS client | specification.md:11 | FR-013, FR-014 | +| Browser | - | - | Transport-agnostic (caller provides) | specification.md:11 | FR-013, FR-014 | | Python | aiohttp | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | | Python | pyarrow | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | -| Dart | nats | Latest | NATS client | specification.md:11 | FR-013, FR-014 | | Dart | http | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | | Dart | uuid | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 | | Dart | dart-arrow | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | -| Rust | nats | Latest | NATS client | specification.md:11 | FR-013, FR-014 | | Rust | serde | Latest | JSON serialization | specification.md:11 | FR-012, NFR-101, NFR-102 | | Rust | serde_json | Latest | JSON handling | specification.md:11 | FR-012, NFR-101, NFR-102 | | Rust | tokio | Latest | Async runtime | specification.md:11 | FR-013, FR-014 | diff --git a/docs/requirements.md b/docs/requirements.md index f6b9198..fc20b68 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -11,7 +11,7 @@ ### 1.1 Business Goal -msghandler is a cross-platform, bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using NATS as the message bus. The system implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB) by uploading them to an HTTP file server instead of sending raw binary data over NATS. +msghandler is a cross-platform, bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. The system implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB) by uploading them to an HTTP file server instead of sending raw binary data over the transport layer. ### 1.2 User Stories (with acceptance criteria) @@ -19,13 +19,13 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless |-------|----------|---------------------| | **As a Julia developer**, I want to send text messages to JavaScript/Dart applications that lives on a server and also on a browser | P1 | Text messages are serialized, encoded, and received correctly across platforms | | **As a Python developer**, I want to send tabular data to Julia/Dart applications | P1 | DataFrame exchange works with both Arrow IPC and JSON formats | -| **As a JavaScript developer**, I want to send large files (>0.5MB) from JavaScript applications that lives on a server and also on a browser to other applications | P1 | Large files are automatically uploaded to file server and URLs are sent via NATS | +| **As a JavaScript developer**, I want to send large files (>0.5MB) from JavaScript applications that lives on a server and also on a browser to other applications | P1 | Large files are automatically uploaded to file server and URLs are sent via the transport layer | | **As a Dart developer**, I want to send text messages to other platforms | P1 | Text messages are serialized, encoded, and received correctly across platforms | | **As a Dart developer**, I want to send dictionary data to other platforms | P1 | JSON-serializable data is exchanged correctly | | **As a Dart developer**, I want to send tabular data (List) to other platforms | P1 | JSON table format exchange works with Arrow IPC on desktop | -| **As a Dart developer**, I want to send large files (>0.5MB) | P1 | Large files are automatically uploaded to file server and URLs are sent via NATS | +| **As a Dart developer**, I want to send large files (>0.5MB) | P1 | Large files are automatically uploaded to file server and URLs are sent via the transport layer | | **As a MicroPython developer**, I want to send sensor data with minimal memory usage | P1 | Direct transport works for payloads <100KB on memory-constrained devices | -| **As a Rust developer**, I want to send and receive messages with type-safe APIs | P1 | Rust implementation uses serde for serialization, tokio for async, and nats-io for NATS connectivity | +| **As a Rust developer**, I want to send and receive messages with type-safe APIs | P1 | Rust implementation uses serde for serialization, tokio for async, and transport-agnostic client for connectivity | | **As a developer**, I want to send mixed-content messages (text + image + file) | P1 | msghandler accepts list of (dataname, data, type) tuples and handles each payload appropriately | | **As a developer**, I want to receive multi-payload messages | P1 | msghandler returns payloads as list of tuples with correct types preserved | | **As a developer**, I want to use Plik as the file server | P2 | Plik one-shot upload mode is supported with upload ID and token handling | @@ -59,37 +59,32 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | File server integration | Plik one-shot upload and custom HTTP server support | | Reliability features | Exponential backoff retry and correlation ID propagation | | Message serialization | Converts data types to binary format (Base64, JSON, Arrow IPC) | -| NATS communication | Publishing and subscription via NATS subjects | +| Transport communication | Publishing and subscription via message broker (NATS, MQTT, WebSocket, etc.) | ### 2.2 Out of Scope | Feature | Reason | |---------|--------| -| NATS JetStream support | Core NATS sufficient for current use cases | +| Advanced transport features | Basic transport sufficient for current use cases | | Message compression | Compression adds complexity without clear benefit | | Message encryption | Payload encryption is application-layer concern | -| Persistent message queues | NATS request-reply pattern sufficient | -| Advanced routing rules | Simple NATS subject matching sufficient | +| Persistent message queues | Transport request-reply pattern sufficient | +| Advanced routing rules | Simple topic matching sufficient | ### 2.3 Dependencies | Platform | Package | Version | |----------|---------|---------| -| Julia | NATS.jl | Latest stable | | Julia | JSON.jl | Latest stable | | Julia | Arrow.jl | Latest stable | | Julia | HTTP.jl | Latest stable | | Julia | UUIDs.jl | Latest stable | -| Node.js | nats | Latest stable | | Node.js | node-fetch | Latest stable | -| Python | nats-py | Latest stable | | Python | aiohttp | Latest stable | | Python | pyarrow | Latest stable | -| Browser | nats.ws | Latest stable | -| Dart | nats | Latest stable | +| Browser | - | Transport-agnostic (caller provides) | | Dart | http | Latest stable | | Dart | uuid | Latest stable | -| Rust | nats | Latest stable | | Rust | serde | Latest stable | | Rust | serde_json | Latest stable | | Rust | tokio | Latest stable | @@ -100,7 +95,7 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | Platform | Minimum Version | Notes | |----------|-----------------|-------| | Julia | 1.7+ | Arrow.jl required for arrowtable support | -| Node.js | 16+ | nats.js required, Arrow IPC supported | +| Node.js | 16+ | Transport client required, Arrow IPC supported | | Python | 3.8+ | pyarrow required for arrowtable support | | Browser | Latest | No Arrow IPC (uses jsontable only) | | Dart | 2.17+ | Supports Desktop (Dart SDK), Flutter (Dart SDK), and Web (Dart SDK) | @@ -115,8 +110,8 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless |----|-------------|-------------| | **FR-001** | Cross-platform text messaging | System shall allow users to send text messages between Julia, JavaScript, Python, and MicroPython applications | | **FR-002** | Cross-platform tabular data | System shall support DataFrame exchange between Julia and Python applications using Arrow IPC format | -| **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via NATS | -| **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via NATS without file server upload | +| **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via transport | +| **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via transport without file server upload | | **FR-005** | MicroPython support | System shall support payloads <100KB on MicroPython devices using direct transport | | **FR-006** | Multi-payload messages | System shall accept and process lists of (dataname, data, type) tuples | | **FR-007** | Payload type preservation | System shall preserve payload types when returning multi-payload messages | @@ -125,8 +120,8 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | **FR-010** | Exponential backoff retry | System shall implement exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) for file server download failures | | **FR-011** | Correlation ID propagation | System shall propagate correlation IDs through all message processing steps | | **FR-012** | Message serialization | System shall serialize data types using Base64, JSON, or Arrow IPC encoding | -| **FR-013** | NATS publishing | System shall return JSON string representation for caller to publish to NATS subjects (caller is responsible for actual NATS publish) | -| **FR-014** | NATS subscription | System shall receive and process NATS messages by accepting JSON string from NATS payload | +| **FR-013** | Transport publishing | System shall return JSON string representation for caller to publish via transport layer (caller is responsible for actual transport publish) | +| **FR-014** | Transport subscription | System shall receive and process messages by accepting JSON string from transport payload | --- @@ -138,10 +133,10 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless |----|-------------|---------------|-------------| | **NFR-101** | Message serialization overhead | <50ms for 10KB payload | Benchmark tests | | **NFR-102** | Message deserialization overhead | <50ms for 10KB payload | Benchmark tests | -| **NFR-103** | NATS connection establishment | <100ms | Connection pool benchmarks | +| **NFR-103** | Transport connection establishment | <100ms | Connection pool benchmarks | | **NFR-104** | File upload latency | <1s for 0.5MB file | Integration tests | | **NFR-105** | File download latency | <1s for 0.5MB file | Integration tests | -| **NFR-106** | Concurrent connections | Support 100+ simultaneous NATS connections | Scale testing | +| **NFR-106** | Concurrent connections | Support 100+ simultaneous transport connections | Scale testing | | **NFR-107** | Message throughput | Handle 1000+ messages/second per instance | Load testing | | **NFR-108** | File server scalability | Support horizontal scaling of file server backend | Architecture review | @@ -149,16 +144,16 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | ID | Requirement | Specification | |----|-------------|---------------| -| **NFR-201** | Message delivery | At-least-once delivery semantics via NATS | +| **NFR-201** | Message delivery | At-least-once delivery semantics via transport | | **NFR-202** | File server availability | Graceful degradation when file server is unavailable | -| **NFR-203** | Connection recovery | Auto-reconnect on NATS connection failure | +| **NFR-203** | Connection recovery | Auto-reconnect on transport connection failure | ### 4.3 Privacy & Security | ID | Requirement | Specification | |----|-------------|---------------| | **NFR-301** | Payload integrity | SHA-256 checksum support via metadata | -| **NFR-302** | Transport security | TLS support for NATS connections | +| **NFR-302** | Transport security | TLS support for transport connections | | **NFR-303** | File server security | Authentication token for file uploads | ### 4.4 Observability & Telemetry @@ -234,11 +229,11 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | Platform | Maximum | Notes | |----------|---------|-------| -| Desktop | Unlimited | Limited by NATS server configuration | -| Dart Desktop | Unlimited | Limited by NATS server configuration | -| Dart Flutter | Unlimited | Limited by NATS server configuration | -| Dart Web | Unlimited | Limited by NATS server configuration | -| Rust | Unlimited | Limited by NATS server configuration | +| Desktop | Unlimited | Limited by transport server configuration | +| Dart Desktop | Unlimited | Limited by transport server configuration | +| Dart Flutter | Unlimited | Limited by transport server configuration | +| Dart Web | Unlimited | Limited by transport server configuration | +| Rust | Unlimited | Limited by transport server configuration | | MicroPython | 50KB | Hard limit due to 256KB-1MB memory | --- @@ -252,7 +247,7 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | `correlation_id` | String (UUID) | Track message flow across systems | | `msg_id` | String (UUID) | Unique message identifier | | `timestamp` | String (ISO 8601) | Message publication timestamp | -| `send_to` | String | NATS subject to publish to | +| `send_to` | String | Topic/subject to publish to | | `msg_purpose` | String | ACK, NACK, updateStatus, shutdown, chat | | `sender_name` | String | Sender application name | | `sender_id` | String (UUID) | Sender unique identifier | @@ -260,7 +255,7 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | `receiver_id` | String (UUID) | Receiver unique identifier (empty = broadcast) | | `reply_to` | String | Topic for reply messages | | `reply_to_msg_id` | String | Message ID being replied to | -| `broker_url` | String | NATS server URL | +| `broker_url` | String | Broker URL for the transport layer | | `metadata` | Dict | Message-level metadata | | `payloads` | Array | List of payload objects | @@ -289,14 +284,14 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | `Failed to upload` | File server error | Throw error | | `Failed to fetch` | File server unavailable | Retry with exponential backoff | | `Unknown transport` | Invalid transport type | Throw error | -| `NATS connection failed` | NATS unavailable | Throw error | +| `Transport connection failed` | Transport/broker unavailable | Throw error | ### 9.2 Exception Handling | Scenario | Handler | |----------|---------| | File server unavailable | Retry up to 5 times with exponential backoff | -| NATS publish failure | Connection auto-reconnect | +| Transport publish failure | Handled by caller | | Deserialization error | Log correlation ID and throw error | | Memory overflow (MicroPython) | Reject payloads >50KB | @@ -350,7 +345,7 @@ function smartsend( )::Tuple{msg_envelope_v1, String} where {T1<:Any} ``` -**Note**: NATS publishing is the caller's responsibility. `smartsend` returns `(env::msg_envelope_v1, env_json_str::String)`. +**Note**: Publishing via the transport layer is the caller's responsibility. `smartsend` returns `(env::msg_envelope_v1, env_json_str::String)`. ### 11.2 smartreceive Signature @@ -364,7 +359,9 @@ function smartreceive( )::JSON.Object{String, Any} ``` -**Note**: Pass `String(nats_msg.payload)` from NATS subscription to `smartreceive`. +**Note**: Pass the payload string from the transport subscription to `smartreceive`. The input is the JSON string payload from the transport message, not the transport message object directly. + +**Note**: Pass the payload from the transport subscription to `smartreceive`. --- @@ -374,7 +371,7 @@ function smartreceive( | Component | Minimum | Notes | |-----------|---------|-------| -| NATS Server | 1 instance | Single node for development | +| Message Broker | 1 instance | Single node for development | | File Server | 1 instance | HTTP server for large payloads | | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) | | Client Memory | 256KB | MicroPython devices | @@ -383,7 +380,7 @@ function smartreceive( | Variable | Default | Description | |----------|---------|-------------| -| `NATS_URL` | `nats://localhost:4222` | NATS server URL | +| `BROKER_URL` | `ws://localhost:4222` | Message broker URL | | `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL | | `SIZE_THRESHOLD` | `500000` | Size threshold in bytes (0.5MB) | @@ -409,6 +406,10 @@ function smartreceive( | Date | Version | Changes | |------|---------|---------| +| 2026-05-15 | 1.3.0 | Made transport layer agnostic | +| - | - | Removed all NATS-specific dependencies and references | +| - | - | Updated all NATS references to generic "transport layer"/"message broker" | +| - | - | Removed NATS client packages from dependencies tables | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | | - | - | Fixed smartsend signature: removed is_publish, NATS_connection; added sender_name | | - | - | Fixed smartreceive signature: takes msg_json_str::String instead of msg::NATS.Msg | diff --git a/docs/specification.md b/docs/specification.md index f3b2231..2b32c91 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -10,7 +10,7 @@ ## 1. Technical Contract Overview -This document defines the **technical contract** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using NATS as the message bus. +This document defines the **technical contract** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. This specification serves as the single source of truth for: - **Inputs**: What data structures are accepted by `smartsend()` @@ -28,7 +28,7 @@ This specification serves as the single source of truth for: | Section 5 (Enumerations) | FR-003, FR-004, FR-006, NFR-101 | Enumerations for transport and encoding | | Section 6 (Transport Protocols) | FR-003, FR-004, NFR-104, NFR-105 | Direct and link transport protocols | | Section 7 (Size Thresholds) | FR-004, FR-005, NFR-104, NFR-105 | Size thresholds for transport selection | -| Section 8 (NATS Subject Convention) | FR-013, FR-014 | NATS subject naming patterns | +| Section 8 (Topic Convention) | FR-013, FR-014 | Topic/subject naming patterns | | Section 9 (Error Handling) | FR-010, FR-011, NFR-201, NFR-202, NFR-203 | Error codes and exception handling | | Section 10 (Serialization Rules) | FR-001, FR-002, FR-003, FR-012, NFR-101, NFR-102 | Serialization and encoding rules | | Section 11 (API Contract) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Function signatures for all platforms | @@ -95,7 +95,7 @@ This specification serves as the single source of truth for: | `correlation_id` | `string` | Yes | UUID v4 format | Track message flow across distributed systems | FR-011, NFR-401 | | `msg_id` | `string` | Yes | UUID v4 format | Unique identifier for this specific message | FR-012, NFR-401 | | `timestamp` | `string` | Yes | ISO 8601 UTC | Message publication timestamp | FR-012, NFR-401 | -| `send_to` | `string` | Yes | Non-empty string | NATS subject/topic to publish the message to | FR-013 | +| `send_to` | `string` | Yes | Non-empty string | Topic/subject to publish the message to | FR-013 | | `msg_purpose` | `string` | Yes | Enum | Purpose of the message | FR-013 | | `sender_name` | `string` | Yes | Non-empty string | Name of the sender application | FR-013 | | `sender_id` | `string` | Yes | UUID v4 format | Unique identifier for the sender | FR-013 | @@ -103,7 +103,7 @@ This specification serves as the single source of truth for: | `receiver_id` | `string` | Yes | Any string | UUID of the receiver (empty = broadcast) | FR-013 | | `reply_to` | `string` | Yes | Any string | Topic where receiver should reply | FR-013 | | `reply_to_msg_id` | `string` | Yes | Any string | Message ID this message is replying to | FR-013 | -| `broker_url` | `string` | Yes | Valid URL | NATS broker URL | FR-013 | +| `broker_url` | `string` | Yes | Valid URL | Broker URL for the transport layer | FR-013 | | `metadata` | `object` | No | Any JSON object | Message-level metadata | NFR-401 | | `payloads` | `array` | Yes | Non-empty array | List of payload objects | FR-012, FR-013 | @@ -240,7 +240,7 @@ await smartsend("/agent/v1/process", data) | Value | Description | Data Format | Use Case | |-------|-------------|-------------|----------| -| `direct` | Payload sent directly via NATS | Base64-encoded string | Payloads < size_threshold | +| `direct` | Payload sent directly via the transport layer | Base64-encoded string | Payloads < size_threshold | | `link` | Payload uploaded to file server | HTTP URL | Payloads ≥ size_threshold | ### `encoding` Enum @@ -312,7 +312,7 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa --- -## NATS Subject Convention +## Topic Convention ### Subject Naming Pattern @@ -357,7 +357,7 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa | `INVALID_TRANSPORT` | 400 | Unsupported transport type | Use `direct` or `link` | FR-003, FR-004, FR-006 | | `UPLOAD_FAILED` | 500 | File server upload failed | Retry or use direct transport | FR-008, FR-009 | | `DOWNLOAD_FAILED` | 503 | File server download failed | Retry with exponential backoff | FR-010, FR-011, NFR-201, NFR-202 | -| `NATS_CONNECTION_FAILED` | 503 | NATS connection failed | Check NATS server availability | FR-013, FR-014, NFR-201, NFR-203 | +| `TRANSPORT_CONNECTION_FAILED` | 503 | Transport connection failed | Check broker/server availability | FR-013, FR-014, NFR-201, NFR-203 | | `DESERIALIZATION_ERROR` | 500 | Payload deserialization failed | Check payload_type matches data | FR-001, FR-002, FR-003, FR-012 | | `SIZE_EXCEEDED` | 413 | Payload exceeds maximum size | Split payload or use link transport | FR-003, FR-004, FR-005, NFR-104, NFR-105 | @@ -366,7 +366,7 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa | Scenario | Handler | Retry Policy | Requirement ID | |----------|---------|--------------|----------------| | File server unavailable | Retry up to 5 times | Exponential backoff (100ms → 5000ms) | FR-010, NFR-202 | -| NATS publish failure | Connection auto-reconnect | TCP-level reconnection | FR-013, FR-014, NFR-201, NFR-203 | +| Transport publish failure | Handled by caller | Caller's retry logic | FR-013, FR-014, NFR-201, NFR-203 | | Deserialization error | Log correlation ID and throw | No retry (data corruption) | FR-001, FR-002, FR-003, FR-012, NFR-401 | | Memory overflow (MicroPython) | Reject payloads >50KB | No retry (client-side check) | FR-005, NFR-106 | @@ -435,7 +435,7 @@ function smartsend( )::Tuple{msg_envelope_v1, String} where {T1<:Any} ``` -**Note**: NATS publishing is the caller's responsibility. Returns `(env::msg_envelope_v1, env_json_str::String)`. +**Note**: Publishing via the transport layer is the caller's responsibility. Returns `(env::msg_envelope_v1, env_json_str::String)`. #### Python @@ -443,7 +443,7 @@ function smartsend( async def smartsend( subject: str, data: List[Tuple[str, Any, str]], - broker_url: str = "nats://localhost:4222", + broker_url: str = DEFAULT_BROKER_URL, fileserver_url: str = "http://localhost:8080", fileserver_upload_handler: Callable = plik_oneshot_upload, size_threshold: int = 500_000, @@ -459,7 +459,7 @@ async def smartsend( ) -> Tuple[Dict, str]: ``` -**Note**: NATS publishing is the caller's responsibility. +**Note**: Publishing via the transport layer is the caller's responsibility. #### JavaScript (Node.js) @@ -485,7 +485,7 @@ async function smartsend( ): Promise<[Object, string]>; ``` -**Note**: NATS publishing is the caller's responsibility. +**Note**: Publishing via the transport layer is the caller's responsibility. #### JavaScript (Browser) @@ -511,7 +511,7 @@ async function smartsend( ): Promise<[Object, string]>; ``` -**Note**: NATS publishing is the caller's responsibility. +**Note**: Publishing via the transport layer is the caller's responsibility. #### MicroPython @@ -524,7 +524,7 @@ def smartsend( ) -> Tuple[Dict, str]: ``` -**Note**: NATS publishing is the caller's responsibility. +**Note**: Publishing via the transport layer is the caller's responsibility. #### Dart (Desktop/Flutter) @@ -532,10 +532,10 @@ def smartsend( Future<[Map, String]> smartsend( String subject, List> data, { - String brokerUrl = 'nats://localhost:4222', - String fileserverUrl = 'http://localhost:8080', + String brokerUrl = DEFAULT_BROKER_URL, + String fileserverUrl = DEFAULT_FILESERVER_URL, Function? fileserverUploadHandler, - int sizeThreshold = 500000, + int sizeThreshold = DEFAULT_SIZE_THRESHOLD, String? correlationId, String msgPurpose = 'chat', String senderName = 'msghandler', @@ -547,9 +547,8 @@ Future<[Map, String]> smartsend( String? senderId, }) async { // Returns [envelope, jsonString] - // NATS publishing is caller's responsibility + // Publishing via transport layer is caller's responsibility } -``` #### Dart Web @@ -557,7 +556,7 @@ Future<[Map, String]> smartsend( Future<[Map, String]> smartsend( String subject, List> data, { - String brokerUrl = 'nats://localhost:4222', + String brokerUrl = DEFAULT_BROKER_URL, String fileserverUrl = 'http://localhost:8080', Function? fileserverUploadHandler, int sizeThreshold = 500000, @@ -572,7 +571,7 @@ Future<[Map, String]> smartsend( String? senderId, }) async { // Returns [envelope, jsonString] - // NATS publishing is caller's responsibility + // Publishing via transport layer is caller's responsibility } ``` @@ -635,7 +634,7 @@ pub struct MsgEnvelopeV1 { } ``` -**Note**: NATS publishing is the caller's responsibility. Returns `Result<(MsgEnvelopeV1, String), msghandlerError>`. Uses `serde` for JSON serialization. +**Note**: Publishing via the transport layer is the caller's responsibility. Returns `Result<(MsgEnvelopeV1, String), msghandlerError>`. Uses `serde` for JSON serialization. ### `smartreceive` Function Signature @@ -643,7 +642,7 @@ pub struct MsgEnvelopeV1 { ```julia function smartreceive( - msg_json_str::String; # Pass String(nats_msg.payload) from NATS subscription + msg_json_str::String; # Pass payload from transport subscription fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, base_delay::Int = 100, @@ -651,13 +650,13 @@ function smartreceive( )::JSON.Object{String, Any} ``` -**Note**: Input is JSON string from NATS message payload, not NATS.Msg directly. +**Note**: Input is the JSON string payload from the transport subscription, not the transport message object directly. #### Python ```python async def smartreceive( - msg_json_str: str, # JSON string from NATS message payload + msg_json_str: str, # JSON string from transport message payload fileserver_download_handler: Callable = fetch_with_backoff, max_retries: int = 5, base_delay: int = 100, @@ -665,13 +664,13 @@ async def smartreceive( ) -> Dict[str, Any]: ``` -**Note**: Input is JSON string from NATS message payload. +**Note**: Input is the JSON string payload from the transport message. #### JavaScript (Node.js) ```typescript async function smartreceive( - msg_json_str: string, // JSON string from NATS message payload + msg_json_str: string, // JSON string from transport message payload options?: { fileserver_download_handler?: Function; max_retries?: number; @@ -685,7 +684,7 @@ async function smartreceive( ```typescript async function smartreceive( - msg_json_str: string, // JSON string from NATS message payload + msg_json_str: string, // JSON string from transport message payload options?: { fileserver_download_handler?: Function; max_retries?: number; @@ -695,7 +694,7 @@ async function smartreceive( ): Promise; ``` -**Note**: Input is JSON string from NATS message payload. +**Note**: Input is the JSON string payload from the transport message. #### MicroPython @@ -703,13 +702,13 @@ async function smartreceive( def smartreceive(msg_json_str: str, **kwargs) -> Dict[str, Any]: ``` -**Note**: Input is JSON string from NATS message payload. +**Note**: Input is the JSON string payload from the transport message. #### Dart (Desktop/Flutter) ```dart Future> smartreceive( - Map msg_json_str, // JSON object from NATS message payload + Map msg_json_str, // JSON object from transport message payload { Function? fileserverDownloadHandler, int maxRetries = 5, @@ -724,7 +723,7 @@ Future> smartreceive( ```dart Future> smartreceive( - Map msg_json_str, // JSON object from NATS message payload + Map msg_json_str, // JSON object from transport message payload { Function? fileserverDownloadHandler, int maxRetries = 5, @@ -739,7 +738,7 @@ Future> smartreceive( ```rust pub async fn smartreceive( - msg_json_str: &str, // JSON string from NATS message payload + msg_json_str: &str, // JSON string from transport message payload options: &SmartreceiveOptions, ) -> Result @@ -752,7 +751,7 @@ pub struct SmartreceiveOptions { } ``` -**Note**: Input is JSON string from NATS message payload. Returns `Result`. +**Note**: Input is the JSON string payload from the transport message. Returns `Result`. --- @@ -897,7 +896,7 @@ function fileserver_download_handler( |------|----------|----------|-------| | [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | Ground truth implementation | | [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | Server-side JavaScript | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only, WebSocket NATS | Client-side rendering | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | Client-side rendering | | [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | Desktop Python | | [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | Desktop/Flutter/Web | | [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe | Uses tokio + serde + arrow2 | @@ -910,7 +909,7 @@ The browser implementation ([`src/msghandler_csr.js`](../src/msghandler_csr.js)) | Constraint | Reason | Workaround | |------------|--------|------------| | No Apache Arrow IPC | Browser-incompatible dependency | Use `jsontable` for tabular data | -| WebSocket NATS only | Browser cannot use TCP directly | Use `ws://` or `wss://` broker URLs | +| WebSocket only | Browser cannot use TCP directly | Use `ws://` or `wss://` broker URLs | | Fetch API for HTTP | Browser fetch() API only | Compatible with Plik and other HTTP servers | ### Payload Type Availability by Platform @@ -941,7 +940,7 @@ flowchart TD D --> F[Build envelope with metadata] E --> F F --> G[Convert envelope to JSON string] - G --> H[Publish to NATS subject] + G --> H[Publish to topic via transport] H --> I[Return envelope and JSON string to caller] style A fill:#f9f9f9,stroke:#333 @@ -954,7 +953,7 @@ flowchart TD ```mermaid flowchart TD - A[NATS message arrives] --> B[Parse JSON envelope] + A[Transport message arrives] --> B[Parse JSON envelope] B --> C[For each payload: Check transport type] C -->|transport == direct| D[Direct Transport: Extract Base64] C -->|transport == link| E[Link Transport: Fetch from URL] @@ -1046,23 +1045,17 @@ flowchart TD | Platform | Package | Version | Purpose | |----------|---------|---------|---------| -| Julia | NATS.jl | Latest | NATS client | | Julia | JSON.jl | Latest | JSON serialization | | Julia | Arrow.jl | Latest | Arrow IPC support | | Julia | HTTP.jl | Latest | HTTP file server | | Julia | UUIDs.jl | Latest | UUID generation | -| Node.js | nats | Latest | NATS client (TCP) | | Node.js | node-fetch | Latest | HTTP file server | -| Browser | nats.ws | Latest | NATS client (WebSocket) | -| Browser | nats | Latest | NATS client (for bundling) | -| Python | nats-py | Latest | NATS client | +| Browser | - | - | Transport-agnostic (caller provides) | | Python | aiohttp | Latest | HTTP file server | | Python | pyarrow | Latest | Arrow IPC support | -| Dart | nats | Latest | NATS client | | Dart | http | Latest | HTTP file server | | Dart | uuid | Latest | UUID generation | | Dart | dart-arrow | Latest | Arrow IPC support (Desktop/Flutter) | -| Rust | nats | Latest | NATS client | | Rust | serde | Latest | JSON serialization | | Rust | serde_json | Latest | JSON handling | | Rust | tokio | Latest | Async runtime | @@ -1084,6 +1077,10 @@ flowchart TD | Date | Version | Changes | |------|---------|---------| +| 2026-05-15 | 1.3.0 | Made transport layer agnostic | +| - | - | Removed all NATS-specific dependencies (NATS.jl, nats, nats-py, nats.ws) | +| - | - | Updated docs to reference generic message broker/transport | +| - | - | broker_url is now metadata only, not used for active connections | | 2026-03-15 | 1.1.0 | Browser connection management | | - | - | Added NATSClient class with keepAlive support | | - | - | Added NATSConnectionPool for connection reuse | @@ -1119,7 +1116,7 @@ flowchart TD |------|----------|----------|--------------------------| | [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only, WebSocket NATS | FR-001 through FR-014, NFR-101 through NFR-405 | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | | [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe | FR-001 through FR-014, NFR-101 through NFR-405 | @@ -1129,23 +1126,17 @@ flowchart TD | Platform | Package | Version | Purpose | Requirements Traceability | |----------|---------|---------|---------|--------------------------| -| Julia | NATS.jl | Latest | NATS client | FR-013, FR-014, NFR-201 | | Julia | JSON.jl | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | | Julia | Arrow.jl | Latest | Arrow IPC support | FR-002, FR-012 | | Julia | HTTP.jl | Latest | HTTP file server | FR-008, FR-009 | | Julia | UUIDs.jl | Latest | UUID generation | FR-011, NFR-401 | -| Node.js | nats | Latest | NATS client (TCP) | FR-013, FR-014 | | Node.js | node-fetch | Latest | HTTP file server | FR-008, FR-009 | -| Browser | nats.ws | Latest | NATS client (WebSocket) | FR-013, FR-014 | -| Browser | nats | Latest | NATS client (for bundling) | FR-013, FR-014 | -| Python | nats-py | Latest | NATS client | FR-013, FR-014 | +| Browser | - | - | Transport-agnostic (caller provides) | FR-013, FR-014 | | Python | aiohttp | Latest | HTTP file server | FR-008, FR-009 | | Python | pyarrow | Latest | Arrow IPC support | FR-002, FR-012 | -| Dart | nats | Latest | NATS client | FR-013, FR-014 | | Dart | http | Latest | HTTP file server | FR-008, FR-009 | | Dart | uuid | Latest | UUID generation | FR-011, NFR-401 | | Dart | dart-arrow | Latest | Arrow IPC support | FR-002, FR-012 | -| Rust | nats | Latest | NATS client | FR-013, FR-014 | | Rust | serde | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | | Rust | serde_json | Latest | JSON handling | FR-012, NFR-101, NFR-102 | | Rust | tokio | Latest | Async runtime | FR-013, FR-014 | @@ -1160,6 +1151,10 @@ flowchart TD | Date | Version | Changes | Requirement ID(s) | |------|---------|---------|-------------------| +| 2026-05-15 | 1.3.0 | Made transport layer agnostic | All | +| - | - | Removed NATS-specific dependencies and references from all docs | All | +| - | - | Updated all NATS references to generic "transport layer"/"message broker" | All | +| - | - | Removed NATS client packages from dependencies tables | All | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All | | - | - | Updated smartsend signatures: removed is_publish, nats_connection; added sender_name | FR-001 through FR-014 | | - | - | Updated smartreceive signatures: takes msg_json_str::String instead of msg | FR-001 through FR-014 | @@ -1200,7 +1195,7 @@ flowchart TD "send_to": { "type": "string", "minLength": 1, - "description": "NATS subject to publish to" + "description": "Topic/subject to publish to" }, "msg_purpose": { "type": "string", @@ -1236,8 +1231,7 @@ flowchart TD }, "broker_url": { "type": "string", - "pattern": "^nats://[^\\s]+$", - "description": "NATS broker URL" + "description": "Broker URL for the transport layer" }, "metadata": { "type": "object", @@ -1302,14 +1296,14 @@ flowchart TD } ``` -### B. AsyncAPI Specification (NATS) +### B. AsyncAPI Specification ```yaml asyncapi: '2.6.0' info: title: msghandler API version: '1.0.0' - description: Cross-platform bi-directional data bridge using NATS + description: Cross-platform bi-directional data bridge using a message broker contact: name: msghandler Team url: https://github.com/your-org/msghandler @@ -1331,12 +1325,12 @@ channels: schema: type: string publish: - summary: Publish message to NATS + summary: Publish message to transport operationId: publishMessage message: $ref: '#/components/message' subscribe: - summary: Subscribe to NATS messages + summary: Subscribe to messages from transport operationId: subscribeMessage message: $ref: '#/components/message' diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 335fad8..a489997 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -9,7 +9,7 @@ ## 1. Executive Summary -This document provides the **end-to-end trace** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using NATS as the message bus. +This document provides the **end-to-end trace** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. This walkthrough serves as the primary onboarding guide for new developers and explains: - **User scenarios** - Real-world use cases from developer perspective @@ -51,7 +51,7 @@ flowchart TB S3["Size Check"] S4["Transport Selection"] S5["Build Envelope"] - S6["Publish to NATS"] + S6["Publish to transport"] S1 --> S2 S2 --> S3 @@ -62,7 +62,7 @@ flowchart TB subgraph Receiver["Receiver (smartreceive)"] direction LR - R1["Subscribe to NATS"] + R1["Subscribe via transport"] R2["Parse Envelope"] R3["Check Transport"] R4["Deserialize Data"] @@ -99,7 +99,7 @@ flowchart TB | Principle | Description | Rationale | |-----------|-------------|-----------| -| **Claim-Check Pattern** | Large payloads uploaded to HTTP server, URL sent via NATS | NATS has message size limits; avoids NATS overflow | +| **Claim-Check Pattern** | Large payloads uploaded to HTTP server, URL sent via transport | Transport has message size limits; avoids overflow | | **Automatic Transport Selection** | Direct (< threshold) vs Link (≥ threshold) based on size | Optimizes memory vs network I/O trade-off | | **Cross-Platform API** | Consistent `smartsend()`/`smartreceive()` across all platforms | Simplifies developer experience | | **Exponential Backoff** | Retry downloads with increasing delays | Handles transient failures gracefully | @@ -148,7 +148,7 @@ For each payload, msghandler determines transport: **Rationale**: - Direct transport is faster for small payloads (no file server round-trip) -- Link transport is used when payload ≥ 0.5MB (avoids NATS size limits) +- Link transport is used when payload ≥ 0.5MB (avoids transport size limits) #### Step 3: Serialization and Encoding @@ -213,25 +213,26 @@ msghandler builds the message envelope: - **reply_to**: Tells backend where to send response - **payloads array**: Contains all data with metadata for proper handling -#### Step 5: Publish to NATS (Caller's Responsibility) +#### Step 5: Publish to Transport (Caller's Responsibility) ```javascript -// NATS publishing is the caller's responsibility -const conn = await NATS.connect({ servers: "ws://localhost:4222" }); -await conn.publish("/agent/wine/api/v1/prompt", msgJson); +// Publishing via the transport layer is the caller's responsibility +// Example with any transport (NATS, MQTT, WebSocket, etc.) +// const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); +// await conn.publish("/agent/wine/api/v1/prompt", msgJson); ``` **Rationale**: -- NATS provides low-latency message delivery +- The transport layer provides message delivery (NATS, MQTT, WebSocket, etc.) - JSON format ensures cross-platform compatibility -- `smartsend()` returns `(env, msgJson)` - caller handles publishing +- `smartsend()` returns `(env, msgJson)` - caller handles publishing via their chosen transport #### Step 6: Julia Backend Receives Message ```julia # Julia backend -nats_msg = NATS.subscription.next() # Get message from NATS -env = smartreceive(String(nats_msg.payload)) +transport_msg = transport_subscription.next() # Get message from transport +env = smartreceive(String(transport_msg.payload)) # env["payloads"] is now: # [ @@ -302,7 +303,7 @@ const [env, msgJson] = await msghandler.smartsend( **Rationale**: - Link transport used for large payloads - File server handles large file upload -- NATS only sends URL (small message) +- Transport only sends URL (small message) #### Step 3: File Server Upload @@ -356,8 +357,8 @@ const response = await plikOneshotUpload( ```julia # Julia backend -nats_msg = NATS.subscription.next() -env = smartreceive(String(nats_msg.payload)) +transport_msg = transport_subscription.next() +env = smartreceive(String(transport_msg.payload)) # msghandler automatically: # 1. Extracts URL from payload @@ -396,7 +397,7 @@ df = pd.DataFrame({ env, msg_json = await smartsend( "/agent/wine/api/v1/analyze", [("data", df, "arrowtable")], - broker_url="nats://localhost:4222", + broker_url=DEFAULT_BROKER_URL, receiver_name="agent-backend" ) ``` @@ -429,8 +430,8 @@ arrow_bytes = buf.getvalue() ```julia # Julia backend -nats_msg = NATS.subscription.next() -env = smartreceive(String(nats_msg.payload)) +transport_msg = transport_subscription.next() +env = smartreceive(String(transport_msg.payload)) # env["payloads"][1] is now: # ("data", DataFrame with id, name, score columns, "arrowtable") @@ -479,7 +480,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; #[tokio::main] async fn main() { - let conn = nats::connect("nats://localhost:4222").unwrap(); + let conn = transport_client::connect("ws://localhost:4222").unwrap(); // Subscribe and receive messages let mut sub = conn.subscribe("/agent/wine/api/v1/analyze").unwrap(); @@ -520,7 +521,7 @@ async fn main() { **Rationale**: - **serde serialization**: Automatic JSON deserialization to `MsgEnvelopeV1` -- **tokio runtime**: Efficient async I/O for NATS and HTTP operations +- **tokio runtime**: Efficient async I/O for transport and HTTP operations - **smartreceive deserialization**: Payload data is deserialized and stored as strings in `payload.data` - **Type dispatch**: `payload_type` field determines how to interpret the `data` string @@ -548,14 +549,14 @@ let (envelope, json_str) = smartsend( ), ], &SmartsendOptions { - broker_url: "nats://localhost:4222".to_string(), + broker_url: DEFAULT_BROKER_URL.to_string(), reply_to: "/python/worker/v1/results".to_string(), msg_purpose: "chat".to_string(), ..Default::default() }, ).await?; -// Caller publishes to NATS +// Caller publishes via transport conn.publish("/agent/wine/api/v1/results", &json_str)?; ``` @@ -569,7 +570,7 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; ```python # Python backend receives Rust response -env = await smartreceive(str(nats_msg.payload)) +env = await smartreceive(str(transport_msg.payload)) # env["payloads"][0] is now: # ("results", arrow_table_data, "arrowtable") @@ -598,9 +599,9 @@ let (envelope, json_str) = smartsend( ), ], &SmartsendOptions { - broker_url: "nats://localhost:4222".to_string(), - fileserver_url: "http://localhost:8080".to_string(), - size_threshold: 500_000, // 0.5MB triggers link transport + broker_url: DEFAULT_BROKER_URL.to_string(), + fileserver_url: DEFAULT_FILESERVER_URL.to_string(), + size_threshold: DEFAULT_SIZE_THRESHOLD, // threshold triggers link transport ..Default::default() }, ).await?; @@ -637,7 +638,7 @@ sensor_data = { env, msg_json = smartsend( "/sensor/device/v1/readings", [("data", sensor_data, "dictionary")], - broker_url="nats://localhost:4222", + broker_url=DEFAULT_BROKER_URL, size_threshold=100000 # 100KB for MicroPython ) ``` @@ -658,15 +659,15 @@ payload_b64 = base64.b64encode(json_bytes).decode('ascii') **Rationale**: - JSON format for human-readable data -- Base64 for NATS compatibility +- Base64 for transport compatibility - UTF-8 for text encoding #### Step 3: Python Backend Receives ```python # Python backend -nats_msg = await nats_consumer.next() -env = await smartreceive(str(nats_msg.payload)) +transport_msg = await transport_consumer.next() +env = await smartreceive(str(transport_msg.payload)) # env["payloads"][0] is now: # ("data", {"temperature": 25.5, "humidity": 60.0, ...}, "dictionary") @@ -708,14 +709,14 @@ const [env, msgJson] = await msghandler.smartsend( **Rationale**: - Empty `receiver_name` = broadcast to all subscribers - Chat messages often include text + images -- NATS wildcard subscriptions route to correct recipients +- Transport wildcard subscriptions route to correct recipients #### Step 2: Python Backend Receives ```python # Python (Backend) -nats_msg = await nats_consumer.next() -env = await smartreceive(str(nats_msg.payload)) +transport_msg = await transport_consumer.next() +env = await smartreceive(str(transport_msg.payload)) # env["payloads"] is now: # [ @@ -733,8 +734,8 @@ env = await smartreceive(str(nats_msg.payload)) ```julia # Julia (Backend) -nats_msg = NATS.subscription.next() -env = smartreceive(String(nats_msg.payload)) +transport_msg = transport_subscription.next() +env = smartreceive(String(transport_msg.payload)) # env["payloads"] is now: # [ @@ -795,7 +796,7 @@ await msghandler.smartsend( | File server unavailable | `UPLOAD_FAILED` | Fall back to direct transport or smaller payloads | | File server download fails | `DOWNLOAD_FAILED` | Retry with exponential backoff | | Payload type mismatch | `DESERIALIZATION_ERROR` | Validate payload_type matches data | -| NATS connection lost | `NATS_CONNECTION_FAILED` | NATS client auto-reconnects | +| Transport connection lost | `TRANSPORT_CONNECTION_FAILED` | Transport client auto-reconnects | ### Error Response Format @@ -828,14 +829,14 @@ correlation_id = string(uuid4()) # Use throughout the flow log_trace(correlation_id, "Starting smartsend") log_trace(correlation_id, "Serialized payload size: 100 bytes") -log_trace(correlation_id, "Published to NATS") +log_trace(correlation_id, "Published to transport") ``` **Log Format**: ``` [2026-03-13T16:30:00.000Z] [Correlation: abc123...] Starting smartsend [2026-03-13T16:30:00.001Z] [Correlation: abc123...] Serialized payload size: 100 bytes -[2026-03-13T16:30:00.002Z] [Correlation: abc123...] Published to NATS +[2026-03-13T16:30:00.002Z] [Correlation: abc123...] Published to transport ``` --- @@ -846,7 +847,7 @@ log_trace(correlation_id, "Published to NATS") | Strategy | Description | When to Use | |----------|-------------|-------------| -| Pre-create NATS connection | Reuse connection for multiple sends | High-throughput scenarios | +| Pre-create transport connection | Reuse connection for multiple sends | High-throughput scenarios | | Adjust size threshold | Increase threshold if file server slow | File server bottleneck | | Use direct transport | Avoid file server for small payloads | Low latency requirements | @@ -868,7 +869,7 @@ log_trace(correlation_id, "Published to NATS") | Component | Minimum | Notes | |-----------|---------|-------| -| NATS Server | 1 instance | Single node for development | +| Message Broker | 1 instance | Single node for development | | File Server | 1 instance | HTTP server for large payloads | | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) | | Client Memory | 256KB | MicroPython devices | @@ -877,7 +878,7 @@ log_trace(correlation_id, "Published to NATS") | Variable | Default | Description | |----------|---------|-------------| -| `NATS_URL` | `nats://localhost:4222` | NATS server URL | +| `BROKER_URL` | `ws://localhost:4222` | Message broker URL | | `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL | | `SIZE_THRESHOLD` | `500000` | Size threshold in bytes (0.5MB) | @@ -911,7 +912,7 @@ log_trace(correlation_id, "Published to NATS") |------|----------|----------|---------------------------| | [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | specification.md:2-19 (all sections) | | [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | specification.md:2-19 (all sections) | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only, WebSocket NATS | specification.md:2-19 (all sections) | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | specification.md:2-19 (all sections) | | [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | specification.md:2-19 (all sections) | | [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | specification.md:2-19 (all sections) | | [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe, file upload helpers | specification.md:2-19 (all sections) | @@ -923,6 +924,10 @@ log_trace(correlation_id, "Published to NATS") | Date | Version | Changes | Specification Reference | |------|---------|---------|------------------------| +| 2026-05-15 | 1.5.0 | Made transport layer agnostic | All sections | +| - | - | Removed all NATS-specific references from walkthrough | All sections | +| - | - | Updated code examples to use transport-agnostic patterns | All sections | +| - | - | Updated diagrams to remove NATS-specific labels | All sections | | 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartreceive` deserialization changes | All sections | | - | - | `smartreceive` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | | - | - | Added `plik_upload_file` convenience function documentation | specification.md:13 | @@ -932,8 +937,8 @@ log_trace(correlation_id, "Published to NATS") | - | - | Added Rust user scenario (User Scenario 4) | specification.md:11 (Rust API) | | - | - | Updated scenario numbering (MicroPython → Scenario 5, Cross-Platform → Scenario 6) | All sections | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All sections | -| - | - | Updated smartreceive calls to use String(nats_msg.payload) pattern | All sections | -| - | - | Removed NATSClient.publish() calls (caller responsible for NATS publishing) | All sections | +| - | - | Updated smartreceive calls to use transport payload pattern | All sections | +| - | - | Removed NATSClient.publish() calls (caller responsible for transport publishing) | All sections | | - | - | Removed is_publish and nats_connection parameter references | All sections | | 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | All sections | | 2026-03-13 | 1.0.0 | Initial walkthrough documentation | specification.md:2-19 (all sections) | From cc95bc97d3080b5db7b3cf9f6331dd3f8c7978ba Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 15 May 2026 17:40:58 +0700 Subject: [PATCH 03/14] change MsgHandlerError --- AI_prompt.md | 65 +++++++++++ Cargo.lock | 1 + Cargo.toml | 2 +- examples/smartreceive_example.rs | 5 +- examples/smartsend_example.rs | 5 +- src/msghandler.rs | 187 ++++++++++++++----------------- 6 files changed, 153 insertions(+), 112 deletions(-) diff --git a/AI_prompt.md b/AI_prompt.md index 7f0fc22..fcb77a6 100644 --- a/AI_prompt.md +++ b/AI_prompt.md @@ -176,3 +176,68 @@ my server REST API endpoint is sommpanion.yiem.cc/agent-fronent/api/v1/chat but I just placed my custom package for encode and decode message at ./src/msghandler.rs. smartsend() is for encoding and smartreceive() is for decoding. you may also check the file /home/ton/docker-apps/sommpanion/msghandler/docs/walkthrough.md for more info about my package. You can test whether Dioxus webapp can be build using this command "dx bundle --web --release --debug-symbols=false" + + + + + + + + + + +I want to build similar webapp. +My app should be built as client-side-rendering Dioxus-based (version 0.7+). +I already build backend server and I intend to communicate with the webapp using json string that encode the following message envelop: +{ + "correlation_id": "a1b2c3d4...", + "msg_id": "e5f6g7h8...", + "timestamp": "2026-03-13T16:30:00.000Z", + "send_to": "", + "msg_purpose": "chat", + "sender_name": "chat-webapp", + "sender_id": "sender-uuid...", + "receiver_name": "agent-backend", + "receiver_id": "", + "reply_to": "", + "reply_to_msg_id": "", + "broker_url": "myservice.mydomain.com/subservice/api/v1/chat", + "metadata": {}, + "payloads": [ + { + "id": "payload-uuid...", + "dataname": "msg", + "payload_type": "text", + "transport": "direct", + "encoding": "base64", + "size": 20, + "data": "SGVsbG8hIEknIHRlbCB5b3UgSW4gZW5nbGlzaC4=", + "metadata": {"payload_bytes": 20} + }, + { + "id": "payload-uuid...", + "dataname": "avatar", + "payload_type": "image", + "transport": "direct", + "encoding": "base64", + "size": 150000, + "data": "iVBORw0KGgoAAAANSUhEUgAA...", + "metadata": {"payload_bytes": 150000} + }, + { + ..., + "payload_type": "text", + ..., + }, + ... + ] +} +--- +I already have Rust file named msghandler.rs containing the following functions for the webapp to use: +- smartsend() to encode the above message envelop into json string. +- smartreceive() to decode json string back to message envelop. +- the msghandler.rs walkthrough is at /home/ton/docker-apps/sommpanion/msghandler/docs/walkthrough.md +The backend server REST API endpoint is "myservice.mydomain.com/subservice/api/v1/chat". I didn't run the server yet. +I already setup the project structure but you can modify the folder as you see fit. Can you implement the app? Use this command "dx bundle --web --release --debug-symbols=false" to check whether the project can be build. +P.S. AI_prompt.md is for me to use. do not read. + diff --git a/Cargo.lock b/Cargo.lock index b38e325..cf988d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -906,6 +906,7 @@ dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", diff --git a/Cargo.toml b/Cargo.toml index 9c2423f..b675ca2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ path = "src/msghandler.rs" serde = { version = "1", features = ["derive"] } serde_json = "1" tokio = { version = "1", features = ["full"] } -reqwest = { version = "0.12", features = ["json", "stream", "multipart"] } +reqwest = { version = "0.12", features = ["json", "stream", "multipart", "blocking"] } uuid = { version = "1", features = ["v4", "serde"] } base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } diff --git a/examples/smartreceive_example.rs b/examples/smartreceive_example.rs index d5919c6..56a36fd 100644 --- a/examples/smartreceive_example.rs +++ b/examples/smartreceive_example.rs @@ -1,7 +1,6 @@ use msghandler::{smartreceive, SmartreceiveOptions}; -#[tokio::main] -async fn main() { +fn main() { // Simulated message JSON (received via any transport) let msg_json_str = r#"{ "correlation_id": "abc123-def456-ghi789", @@ -43,7 +42,7 @@ async fn main() { let options = SmartreceiveOptions::default(); - match smartreceive(msg_json_str, &options).await { + match smartreceive(msg_json_str, &options) { Ok(envelope) => { println!("=== Envelope Received ==="); println!("Correlation ID: {}", envelope.correlation_id); diff --git a/examples/smartsend_example.rs b/examples/smartsend_example.rs index aade6a1..e145b1f 100644 --- a/examples/smartsend_example.rs +++ b/examples/smartsend_example.rs @@ -1,7 +1,6 @@ use msghandler::{smartsend, Payload, SmartsendOptions}; -#[tokio::main] -async fn main() { +fn main() { // Create mixed payload data let payloads = vec![ ( @@ -33,7 +32,7 @@ async fn main() { ..Default::default() }; - match smartsend("/agent/wine/api/v1/prompt", &payloads, &options).await { + match smartsend("/agent/wine/api/v1/prompt", &payloads, &options) { Ok((envelope, json_str)) => { println!("=== Envelope Created ==="); println!("Correlation ID: {}", envelope.correlation_id); diff --git a/src/msghandler.rs b/src/msghandler.rs index eb0dec4..102ca27 100644 --- a/src/msghandler.rs +++ b/src/msghandler.rs @@ -17,10 +17,9 @@ // Supported types: "text", "dictionary", "arrowtable", "jsontable", // "image", "audio", "video", "binary" -use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use chrono::Utc; -use reqwest::Client; +use reqwest::blocking::Client; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; use std::collections::HashMap; @@ -28,7 +27,6 @@ use std::fmt; use std::path::Path; use std::sync::Arc; use std::time::Duration; -use tokio::time::sleep; use uuid::Uuid; // ============================================================================ @@ -59,7 +57,7 @@ pub const DEFAULT_MAX_DELAY: u64 = 5_000; /// Errors that can occur during msghandler operations #[derive(Debug)] -pub enum msghandlerError { +pub enum MsgHandlerError { /// Unsupported or unknown payload type UnknownPayloadType(String), /// File server upload failed @@ -86,34 +84,34 @@ pub enum msghandlerError { InvalidEnvelope(String), } -impl fmt::Display for msghandlerError { +impl fmt::Display for MsgHandlerError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - msghandlerError::UnknownPayloadType(p) => write!(f, "Unknown payload_type: {}", p), - msghandlerError::UploadFailed(msg) => write!(f, "Failed to upload: {}", msg), - msghandlerError::DownloadFailed { url, retries } => { + MsgHandlerError::UnknownPayloadType(p) => write!(f, "Unknown payload_type: {}", p), + MsgHandlerError::UploadFailed(msg) => write!(f, "Failed to upload: {}", msg), + MsgHandlerError::DownloadFailed { url, retries } => { write!(f, "Failed to fetch {} after {} attempts", url, retries) } - msghandlerError::UnknownTransport(t) => write!(f, "Unknown transport type: {}", t), - msghandlerError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg), - msghandlerError::DeserializationError(msg) => { + MsgHandlerError::UnknownTransport(t) => write!(f, "Unknown transport type: {}", t), + MsgHandlerError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg), + MsgHandlerError::DeserializationError(msg) => { write!(f, "Deserialization error: {}", msg) } - msghandlerError::HttpError { status, message } => { + MsgHandlerError::HttpError { status, message } => { write!(f, "HTTP error {}: {}", status, message) } - msghandlerError::IoError(msg) => write!(f, "IO error: {}", msg), - msghandlerError::JsonError(msg) => write!(f, "JSON error: {}", msg), - msghandlerError::Base64Error(msg) => write!(f, "Base64 error: {}", msg), - msghandlerError::SizeExceeded { size, max } => { + MsgHandlerError::IoError(msg) => write!(f, "IO error: {}", msg), + MsgHandlerError::JsonError(msg) => write!(f, "JSON error: {}", msg), + MsgHandlerError::Base64Error(msg) => write!(f, "Base64 error: {}", msg), + MsgHandlerError::SizeExceeded { size, max } => { write!(f, "Payload size {} exceeds max {}", size, max) } - msghandlerError::InvalidEnvelope(msg) => write!(f, "Invalid envelope: {}", msg), + MsgHandlerError::InvalidEnvelope(msg) => write!(f, "Invalid envelope: {}", msg), } } } -impl std::error::Error for msghandlerError {} +impl std::error::Error for MsgHandlerError {} // ============================================================================ // Payload Enum - Type-safe payload data @@ -318,8 +316,8 @@ impl MsgEnvelopeV1 { } /// Convert the envelope to a JSON string for transport - pub fn to_json(&self) -> Result { - serde_json::to_string(self).map_err(|e| msghandlerError::JsonError(e.to_string())) + pub fn to_json(&self) -> Result { + serde_json::to_string(self).map_err(|e| MsgHandlerError::JsonError(e.to_string())) } } @@ -405,16 +403,15 @@ impl Default for SmartreceiveOptions { // ============================================================================ /// Trait for uploading data to a file server -#[async_trait] pub trait FileUploadHandler: Send + Sync { /// Upload data to the file server /// Returns upload ID, file ID, and download URL - async fn upload( + fn upload( &self, file_server_url: &str, dataname: &str, data: &[u8], - ) -> Result; + ) -> Result; } /// Result of a file server upload @@ -431,17 +428,16 @@ pub struct UploadResult { } /// Trait for downloading data from a file server -#[async_trait] pub trait FileDownloadHandler: Send + Sync { /// Download data from a URL with retry logic - async fn download( + fn download( &self, url: &str, max_retries: u32, base_delay: u64, max_delay: u64, correlation_id: &str, - ) -> Result, msghandlerError>; + ) -> Result, MsgHandlerError>; } // ============================================================================ @@ -457,14 +453,13 @@ pub trait FileDownloadHandler: Send + Sync { /// 4. Returns identifiers and download URL pub struct PlikOneshotUploadHandler; -#[async_trait] impl FileUploadHandler for PlikOneshotUploadHandler { - async fn upload( + fn upload( &self, file_server_url: &str, dataname: &str, data: &[u8], - ) -> Result { + ) -> Result { let client = Client::new(); // Step 1: Create one-shot upload session @@ -474,11 +469,10 @@ impl FileUploadHandler for PlikOneshotUploadHandler { .header("Content-Type", "application/json") .json(&session_body) .send() - .await - .map_err(|e| msghandlerError::UploadFailed(format!("Failed to create upload session: {}", e)))?; + .map_err(|e| MsgHandlerError::UploadFailed(format!("Failed to create upload session: {}", e)))?; if !session_resp.status().is_success() { - return Err(msghandlerError::UploadFailed(format!( + return Err(MsgHandlerError::UploadFailed(format!( "Session creation failed with status: {}", session_resp.status() ))); @@ -486,8 +480,7 @@ impl FileUploadHandler for PlikOneshotUploadHandler { let session_json: JsonValue = session_resp .json() - .await - .map_err(|e| msghandlerError::UploadFailed(format!("Failed to parse session response: {}", e)))?; + .map_err(|e| MsgHandlerError::UploadFailed(format!("Failed to parse session response: {}", e)))?; let uploadid = session_json["id"] .as_str() @@ -499,31 +492,30 @@ impl FileUploadHandler for PlikOneshotUploadHandler { .to_string(); if uploadid.is_empty() || uploadtoken.is_empty() { - return Err(msghandlerError::UploadFailed( + return Err(MsgHandlerError::UploadFailed( "Missing uploadid or uploadToken in session response".to_string(), )); } // Step 2: Upload the file as multipart/form-data let upload_url = format!("{}/file/{}", file_server_url, uploadid); - let form = reqwest::multipart::Form::new() + let form = reqwest::blocking::multipart::Form::new() .part( "file", - reqwest::multipart::Part::bytes(data.to_vec()) + reqwest::blocking::multipart::Part::bytes(data.to_vec()) .file_name(dataname.to_string()) .mime_str("application/octet-stream") - .map_err(|e| msghandlerError::UploadFailed(format!("Invalid MIME type: {}", e)))?, + .map_err(|e| MsgHandlerError::UploadFailed(format!("Invalid MIME type: {}", e)))?, ); let resp = client .post(&upload_url) .header("X-UploadToken", &uploadtoken) .multipart(form) .send() - .await - .map_err(|e| msghandlerError::UploadFailed(format!("Upload request failed: {}", e)))?; + .map_err(|e| MsgHandlerError::UploadFailed(format!("Upload request failed: {}", e)))?; if !resp.status().is_success() { - return Err(msghandlerError::UploadFailed(format!( + return Err(MsgHandlerError::UploadFailed(format!( "Upload failed with status: {}", resp.status() ))); @@ -532,8 +524,7 @@ impl FileUploadHandler for PlikOneshotUploadHandler { let status_code = resp.status().as_u16(); let upload_json: JsonValue = resp .json() - .await - .map_err(|e| msghandlerError::UploadFailed(format!("Failed to parse upload response: {}", e)))?; + .map_err(|e| MsgHandlerError::UploadFailed(format!("Failed to parse upload response: {}", e)))?; let fileid = upload_json["id"].as_str().unwrap_or("").to_string(); @@ -564,29 +555,29 @@ impl FileUploadHandler for PlikOneshotUploadHandler { /// 4. Throws error after max_retries are exhausted pub struct BackoffDownloadHandler; -#[async_trait] impl FileDownloadHandler for BackoffDownloadHandler { - async fn download( + fn download( &self, url: &str, max_retries: u32, base_delay: u64, max_delay: u64, correlation_id: &str, - ) -> Result, msghandlerError> { + ) -> Result, MsgHandlerError> { let client = Client::new(); let mut delay = base_delay; for attempt in 1..=max_retries { - match client.get(url).send().await { + match client.get(url).send() { Ok(response) if response.status().is_success() => { log_trace(correlation_id, &format!( "Successfully fetched {} on attempt {}", url, attempt )); - let bytes = response.bytes().await + let bytes = response + .bytes() .map(|b| b.to_vec()) - .map_err(|_e| msghandlerError::DownloadFailed { + .map_err(|_e| MsgHandlerError::DownloadFailed { url: url.to_string(), retries: max_retries, })?; @@ -611,12 +602,12 @@ impl FileDownloadHandler for BackoffDownloadHandler { } if attempt < max_retries { - sleep(Duration::from_millis(delay)).await; + std::thread::sleep(Duration::from_millis(delay)); delay = (delay * 2).min(max_delay); } } - Err(msghandlerError::DownloadFailed { + Err(MsgHandlerError::DownloadFailed { url: url.to_string(), retries: max_retries, }) @@ -629,14 +620,14 @@ impl FileDownloadHandler for BackoffDownloadHandler { /// Serialize payload data according to the specified payload type. /// Returns the raw bytes for the serialized data. -fn serialize_data(payload: &Payload) -> Result, msghandlerError> { +fn serialize_data(payload: &Payload) -> Result, MsgHandlerError> { match payload { Payload::Text(s) => Ok(s.as_bytes().to_vec()), Payload::Dictionary(v) => serde_json::to_vec(v) - .map_err(|e| msghandlerError::DeserializationError(format!("Dictionary serialization failed: {}", e))), + .map_err(|e| MsgHandlerError::DeserializationError(format!("Dictionary serialization failed: {}", e))), Payload::ArrowTable(b) => Ok(b.clone()), Payload::JsonTable(v) => serde_json::to_vec(v) - .map_err(|e| msghandlerError::DeserializationError(format!("JsonTable serialization failed: {}", e))), + .map_err(|e| MsgHandlerError::DeserializationError(format!("JsonTable serialization failed: {}", e))), Payload::Image(b) => Ok(b.clone()), Payload::Audio(b) => Ok(b.clone()), Payload::Video(b) => Ok(b.clone()), @@ -654,18 +645,18 @@ fn deserialize_data( payload_bytes: &[u8], payload_type: &str, _correlation_id: &str, -) -> Result { +) -> Result { match payload_type { "text" => { let text = String::from_utf8(payload_bytes.to_vec()) - .map_err(|e| msghandlerError::DeserializationError(format!("Invalid UTF-8 for text: {}", e)))?; + .map_err(|e| MsgHandlerError::DeserializationError(format!("Invalid UTF-8 for text: {}", e)))?; Ok(Payload::Text(text)) } "dictionary" => { let json_str = String::from_utf8(payload_bytes.to_vec()) - .map_err(|e| msghandlerError::DeserializationError(format!("Invalid UTF-8 for dictionary: {}", e)))?; + .map_err(|e| MsgHandlerError::DeserializationError(format!("Invalid UTF-8 for dictionary: {}", e)))?; let value: JsonValue = serde_json::from_str(&json_str) - .map_err(|e| msghandlerError::DeserializationError(format!("Invalid JSON for dictionary: {}", e)))?; + .map_err(|e| MsgHandlerError::DeserializationError(format!("Invalid JSON for dictionary: {}", e)))?; Ok(Payload::Dictionary(value)) } "arrowtable" => { @@ -673,16 +664,16 @@ fn deserialize_data( } "jsontable" => { let json_str = String::from_utf8(payload_bytes.to_vec()) - .map_err(|e| msghandlerError::DeserializationError(format!("Invalid UTF-8 for jsontable: {}", e)))?; + .map_err(|e| MsgHandlerError::DeserializationError(format!("Invalid UTF-8 for jsontable: {}", e)))?; let value: JsonValue = serde_json::from_str(&json_str) - .map_err(|e| msghandlerError::DeserializationError(format!("Invalid JSON for jsontable: {}", e)))?; + .map_err(|e| MsgHandlerError::DeserializationError(format!("Invalid JSON for jsontable: {}", e)))?; Ok(Payload::JsonTable(value)) } "image" => Ok(Payload::Image(payload_bytes.to_vec())), "audio" => Ok(Payload::Audio(payload_bytes.to_vec())), "video" => Ok(Payload::Video(payload_bytes.to_vec())), "binary" => Ok(Payload::Binary(payload_bytes.to_vec())), - _ => Err(msghandlerError::UnknownPayloadType(payload_type.to_string())), + _ => Err(MsgHandlerError::UnknownPayloadType(payload_type.to_string())), } } @@ -720,13 +711,12 @@ pub fn log_trace(correlation_id: &str, message: &str) { /// - `options`: Configuration options /// /// # Returns -/// - `Result<(MsgEnvelopeV1, String), msghandlerError>` containing the envelope and JSON string +/// - `Result<(MsgEnvelopeV1, String), MsgHandlerError>` containing the envelope and JSON string /// /// # Example /// ```no_run /// use msghandler::{smartsend, Payload, SmartsendOptions}; /// -/// # async fn example() -> Result<(), Box> { /// let (envelope, json_str) = smartsend( /// "/agent/wine/api/v1/prompt", /// &[ @@ -734,17 +724,15 @@ pub fn log_trace(correlation_id: &str, message: &str) { /// ("data".to_string(), Payload::Binary(vec![1, 2, 3]), "binary".to_string()), /// ], /// &SmartsendOptions::default(), -/// ).await?; +/// ).unwrap(); /// /// // Caller publishes via their preferred transport -/// # Ok(()) -/// # } /// ``` -pub async fn smartsend( +pub fn smartsend( subject: &str, data: &[(String, Payload, String)], options: &SmartsendOptions, -) -> Result<(MsgEnvelopeV1, String), msghandlerError> { +) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { let correlation_id = if options.correlation_id.is_empty() { Uuid::new_v4().to_string() } else { @@ -810,8 +798,7 @@ pub async fn smartsend( log_trace(&correlation_id, "Using link transport, uploading to fileserver"); let upload_result = upload_handler - .upload(&options.fileserver_url, dataname, &payload_bytes) - .await?; + .upload(&options.fileserver_url, dataname, &payload_bytes)?; log_trace(&correlation_id, &format!( "Uploaded to URL: {}", upload_result.url @@ -889,14 +876,13 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// - `options`: Configuration options /// /// # Returns -/// - `Result` with deserialized payloads +/// - `Result` with deserialized payloads /// /// # Example /// ```no_run /// use msghandler::{smartreceive, SmartreceiveOptions}; /// use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; /// -/// # async fn example() -> Result<(), Box> { /// let msg_json_str = r#"{"correlation_id":"abc123","msg_id":"msg-uuid", /// "timestamp":"2026-01-01T00:00:00Z","send_to":"/test", /// "msg_purpose":"chat","sender_name":"test","sender_id":"sender-uuid", @@ -907,26 +893,24 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// "data":"SGVsbG8=","metadata":{"payload_bytes":5} /// }]}"#; /// -/// let envelope = smartreceive(msg_json_str, &SmartreceiveOptions::default()).await?; +/// let envelope = smartreceive(msg_json_str, &SmartreceiveOptions::default()).unwrap(); /// /// for payload in &envelope.payloads { /// if payload.transport == "direct" { -/// let decoded = BASE64.decode(&payload.data)?; +/// let decoded = BASE64.decode(&payload.data).unwrap(); /// println!("{}: {}", payload.dataname, String::from_utf8_lossy(&decoded)); /// } else { /// println!("{}: URL={}", payload.dataname, payload.data); /// } /// } -/// # Ok(()) -/// # } /// ``` -pub async fn smartreceive( +pub fn smartreceive( msg_json_str: &str, options: &SmartreceiveOptions, -) -> Result { +) -> Result { // Parse the JSON envelope let mut env: MsgEnvelopeV1 = serde_json::from_str(msg_json_str) - .map_err(|e| msghandlerError::InvalidEnvelope(format!( + .map_err(|e| MsgHandlerError::InvalidEnvelope(format!( "Failed to parse envelope JSON: {}", e )))?; @@ -953,7 +937,7 @@ pub async fn smartreceive( // Decode Base64 payload let payload_bytes = BASE64.decode(&payload.data) - .map_err(|e| msghandlerError::Base64Error(format!( + .map_err(|e| MsgHandlerError::Base64Error(format!( "Base64 decode failed for '{}': {}", dataname, e )))?; @@ -981,8 +965,7 @@ pub async fn smartreceive( options.base_delay, options.max_delay, &correlation_id, - ) - .await?; + )?; // Deserialize based on type and store result back into payload let deserialized = deserialize_data( @@ -995,7 +978,7 @@ pub async fn smartreceive( updated_payloads.push(updated); } unknown => { - return Err(msghandlerError::UnknownTransport(format!( + return Err(MsgHandlerError::UnknownTransport(format!( "Unknown transport type '{}' for payload '{}'", unknown, dataname ))); @@ -1012,11 +995,11 @@ pub async fn smartreceive( // ============================================================================ /// Send a single text payload -pub async fn send_text( +pub fn send_text( subject: &str, text: &str, options: &SmartsendOptions, -) -> Result<(MsgEnvelopeV1, String), msghandlerError> { +) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { smartsend( subject, &[( @@ -1026,15 +1009,14 @@ pub async fn send_text( )], options, ) - .await } /// Send a single dictionary payload -pub async fn send_dictionary( +pub fn send_dictionary( subject: &str, data: &JsonValue, options: &SmartsendOptions, -) -> Result<(MsgEnvelopeV1, String), msghandlerError> { +) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { smartsend( subject, &[( @@ -1044,15 +1026,14 @@ pub async fn send_dictionary( )], options, ) - .await } /// Send a single binary payload -pub async fn send_binary( +pub fn send_binary( subject: &str, data: &[u8], options: &SmartsendOptions, -) -> Result<(MsgEnvelopeV1, String), msghandlerError> { +) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { smartsend( subject, &[( @@ -1062,7 +1043,6 @@ pub async fn send_binary( )], options, ) - .await } // ============================================================================ @@ -1079,25 +1059,22 @@ pub async fn send_binary( /// - `filepath`: Full path to the local file to upload /// /// # Returns -/// - `Result` with uploadid, fileid, and download URL +/// - `Result` with uploadid, fileid, and download URL /// /// # Example /// ```no_run /// use msghandler::plik_upload_file; /// -/// # async fn example() -> Result<(), Box> { -/// let result = plik_upload_file("http://localhost:8080", "./large_file.zip").await?; +/// let result = plik_upload_file("http://localhost:8080", "./large_file.zip").unwrap(); /// println!("Uploaded to: {}", result.url); -/// # Ok(()) -/// # } /// ``` -pub async fn plik_upload_file( +pub fn plik_upload_file( file_server_url: &str, filepath: &str, -) -> Result { +) -> Result { // Read the file from disk - let data = tokio::fs::read(filepath).await - .map_err(|e| msghandlerError::IoError(format!( + let data = std::fs::read(filepath) + .map_err(|e| MsgHandlerError::IoError(format!( "Failed to read file '{}': {}", filepath, e )))?; @@ -1108,7 +1085,7 @@ pub async fn plik_upload_file( .unwrap_or_default(); // Upload using the Plik one-shot handler - PlikOneshotUploadHandler.upload(file_server_url, &dataname, &data).await + PlikOneshotUploadHandler.upload(file_server_url, &dataname, &data) } // ============================================================================ @@ -1123,7 +1100,7 @@ pub async fn plik_upload_file( // - `SmartsendOptions`, `SmartreceiveOptions` - configuration // - `FileUploadHandler`, `FileDownloadHandler` - trait abstractions // - `PlikOneshotUploadHandler`, `BackoffDownloadHandler` - default implementations -// - `msghandlerError` - error type +// - `MsgHandlerError` - error type #[cfg(test)] mod tests { @@ -1204,10 +1181,10 @@ mod tests { #[test] fn test_error_display() { - let err = msghandlerError::UnknownPayloadType("custom_type".to_string()); + let err = MsgHandlerError::UnknownPayloadType("custom_type".to_string()); assert!(format!("{}", err).contains("custom_type")); - let err = msghandlerError::DownloadFailed { + let err = MsgHandlerError::DownloadFailed { url: "http://example.com/file".to_string(), retries: 5, }; From 396e0848da7e54f78c586585922d4e0186224cdf Mon Sep 17 00:00:00 2001 From: narawat Date: Mon, 18 May 2026 19:30:58 +0700 Subject: [PATCH 04/14] rename to smartpack n smartunpack --- AI_prompt.md | 33 ++++++---- Cargo.toml | 8 +-- README.md | 84 ++++++++++++------------ docs/architecture.md | 40 +++++------ docs/requirements.md | 20 +++--- docs/specification.md | 74 ++++++++++----------- docs/walkthrough.md | 78 +++++++++++----------- etc.txt | 12 ++-- examples/smartreceive_example.rs | 6 +- examples/smartsend_example.rs | 6 +- src/msghandler-csr.js | 40 +++++------ src/msghandler.jl | 30 ++++----- src/msghandler.js | 40 +++++------ src/msghandler.py | 32 ++++----- src/msghandler.rs | 64 +++++++++--------- src/msghandler_mpy.py | 32 ++++----- test/test_js_mix_payloads_receiver.js | 6 +- test/test_js_mix_payloads_sender.js | 4 +- test/test_julia_mix_payloads_receiver.jl | 10 +-- test/test_julia_mix_payloads_sender.jl | 10 +-- test/test_py_mix_payloads_sender.py | 8 +-- 21 files changed, 323 insertions(+), 314 deletions(-) diff --git a/AI_prompt.md b/AI_prompt.md index fcb77a6..11ffb4a 100644 --- a/AI_prompt.md +++ b/AI_prompt.md @@ -4,7 +4,7 @@ Scenario 2: The "Deep Dive" Analysis (High Bandwidth)Focus: Large Arrow tables, Scenario 3: Live Audio/Signal Processing (Multimedia & Metadata)Focus: Raw binary, bi-directional streaming, headers for metadata.The Action: The JS client captures a 2-second "chunk" of microphone audio. It needs Julia to perform a Fast Fourier Transform (FFT) or AI transcription.The Flow:JS (Sender): Sends the raw binary WAV/PCM data. It uses transport headers to store the metadata ($fs = 44.1kHz$, $channels = 1$) to keep the payload purely binary.Julia (Receiver): Processes the audio and sends back a JSON result (the transcription) and an Arrow Table (the frequency spectrum data).Project Requirement Met: Bi-directional flow involving mixed media (Audio) and technical results (Arrow). Scenario 4: The "Catch-Up" (Persistence & State Sync)Focus: Message persistence, late-joining consumers, state sync.The Action: Julia is constantly publishing "System Health" updates. The JS dashboard is closed for 10 minutes. When the user re-opens the dashboard, they need to see the last 10 minutes of history.The Flow:Transport (Server): Uses a persistence layer with a Limits retention policy.JS (Consumer): Connects and requests a "Replay" from the last 10 minutes. It receives a mix of direct (small updates) and link (historical snapshots) messages.Project Requirement Met: Temporal decoupling—consumers can receive data that was sent while they were offline. -Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service, using a unified message envelope with Claim-Check pattern for large payloads.⚠️ STRICT ARCHITECTURAL CONSTRAINTS (Non-Negotiable)Transport Strategy (Claim-Check Pattern):Direct Path: If payload is < 1MB, send data directly inside the message envelope (Base64 encoded).Link Path: If payload is > 1MB, upload to a shared HTTP fileserver/store. The message must only contain the metadata and the download URL.Tabular Data Format: * MUST use Apache Arrow IPC Stream for all tables/DataFrames. No CSV or standard JSON-serialization of tables allowed.System Symmetry: * Both services must function as Producers AND Consumers.Modular Elegance: * Implementation must be abstracted into a SmartSend function and a SmartReceive handler. The developer calling these functions should not need to care if the data is going via direct or HTTP link.Technical Stack & Use CasesJulia: Arrow.jl, JSON3.jl, HTTP.jl.Node.js: apache-arrow, native fetch.Scenarios to Support: * Large Data: Sending a 500MB Arrow table from Julia $\rightarrow$ JS.Media: Sending a 5MB WAV file from JS $\rightarrow$ Julia.Signals: Sending small JSON control commands ($< 10KB$) directly in the envelope.Implementation Requirements1. Unified JSON Envelope:Define a schema containing: correlation_id (UUID), type (table/binary/json), transport (direct/link), payload (if direct), and url (if link).2. The Julia Module:Implement SmartSend(subject, data, type): Handles Arrow serialization to an IOBuffer, checks size, and manages HTTP uploads for large blobs.Implement SmartReceive(msg): Parses envelope, handles the HTTP fetch with Exponential Backoff (to avoid race conditions), and restores the DataFrame.Include a basic HTTP.listen server to serve as the temporary storage.3. The JavaScript Module:Implement a symmetric SmartSend using native fetch and apache-arrow.Implement a JetStream P... (line truncated to 2000 chars) +Role: Principal Systems Architect & Lead Software Engineer.Objective: Implement a high-performance, bi-directional data bridge between a Julia service and a JavaScript (Node.js) service, using a unified message envelope with Claim-Check pattern for large payloads.⚠️ STRICT ARCHITECTURAL CONSTRAINTS (Non-Negotiable)Transport Strategy (Claim-Check Pattern):Direct Path: If payload is < 1MB, send data directly inside the message envelope (Base64 encoded).Link Path: If payload is > 1MB, upload to a shared HTTP fileserver/store. The message must only contain the metadata and the download URL.Tabular Data Format: * MUST use Apache Arrow IPC Stream for all tables/DataFrames. No CSV or standard JSON-serialization of tables allowed.System Symmetry: * Both services must function as Producers AND Consumers.Modular Elegance: * Implementation must be abstracted into a smartpack function and a smartunpack handler. The developer calling these functions should not need to care if the data is going via direct or HTTP link.Technical Stack & Use CasesJulia: Arrow.jl, JSON3.jl, HTTP.jl.Node.js: apache-arrow, native fetch.Scenarios to Support: * Large Data: Sending a 500MB Arrow table from Julia $\rightarrow$ JS.Media: Sending a 5MB WAV file from JS $\rightarrow$ Julia.Signals: Sending small JSON control commands ($< 10KB$) directly in the envelope.Implementation Requirements1. Unified JSON Envelope:Define a schema containing: correlation_id (UUID), type (table/binary/json), transport (direct/link), payload (if direct), and url (if link).2. The Julia Module:Implement smartpack(subject, data, type): Handles Arrow serialization to an IOBuffer, checks size, and manages HTTP uploads for large blobs.Implement smartunpack(msg): Parses envelope, handles the HTTP fetch with Exponential Backoff (to avoid race conditions), and restores the DataFrame.Include a basic HTTP.listen server to serve as the temporary storage.3. The JavaScript Module:Implement a symmetric smartpack using native fetch and apache-arrow.Implement a JetStream P... (line truncated to 2000 chars) @@ -25,7 +25,7 @@ Task: Update msghandler.js to reflect recent changes in msghandler.jl and docs Context: msghandler.jl and docs has been updated. Requirements: Source of Truth: Treat the updated msghandler.jl and docs as the definitive source. -API Consistency: Ensure the Main Package API (e.g., smartsend(), publish_message()) uses consistent naming across all three supported languages. +API Consistency: Ensure the Main Package API (e.g., smartpack(), publish_message()) uses consistent naming across all three supported languages. Ecosystem Variance: Low-level native functions (e.g., connect(), JSON.parse()) should follow the conventions of the specific language ecosystem and do not require cross-language consistency. @@ -164,7 +164,14 @@ Check the following files: I would like to expand this package (msghandler) to include Rust support. Now help me update Rust implementation of this package at ./src/msghandler.rs. - + +I updated ./src/msghandler-csr.js. Can you check whether files in ./docs needs to be update? +You should check the files sequencially in the following order: +1) ./docs/requirements.md +2) ./docs/specification.md +3) ./docs/architecture.md +4) ./docs/walkthrough.md + @@ -173,7 +180,7 @@ I want to build a client-side-rendering Dioxus-based chat webapp. Dioxus version 0.7+ should be great. I already populate the current folder for the project. my server REST API endpoint is sommpanion.yiem.cc/agent-fronent/api/v1/chat but I didn't run the server yet. A message format is JSON string. -I just placed my custom package for encode and decode message at ./src/msghandler.rs. smartsend() is for encoding and smartreceive() is for decoding. +I just placed my custom package for encode and decode message at ./src/msghandler.rs. smartpack() is for encoding and smartunpack() is for decoding. you may also check the file /home/ton/docker-apps/sommpanion/msghandler/docs/walkthrough.md for more info about my package. You can test whether Dioxus webapp can be build using this command "dx bundle --web --release --debug-symbols=false" @@ -185,8 +192,7 @@ You can test whether Dioxus webapp can be build using this command "dx bundle -- - -I want to build similar webapp. +Do you know about ChatGPT chat interface? I want to build similar webapp. My app should be built as client-side-rendering Dioxus-based (version 0.7+). I already build backend server and I intend to communicate with the webapp using json string that encode the following message envelop: { @@ -233,11 +239,14 @@ I already build backend server and I intend to communicate with the webapp using ] } --- -I already have Rust file named msghandler.rs containing the following functions for the webapp to use: -- smartsend() to encode the above message envelop into json string. -- smartreceive() to decode json string back to message envelop. +I already have this Rust module ./src/msghandler.rs containing the following functions for the webapp to use: +- smartpack() to encode the above message envelop into json string. +- smartunpack() to decode json string back to message envelop. - the msghandler.rs walkthrough is at /home/ton/docker-apps/sommpanion/msghandler/docs/walkthrough.md -The backend server REST API endpoint is "myservice.mydomain.com/subservice/api/v1/chat". I didn't run the server yet. -I already setup the project structure but you can modify the folder as you see fit. Can you implement the app? Use this command "dx bundle --web --release --debug-symbols=false" to check whether the project can be build. -P.S. AI_prompt.md is for me to use. do not read. +MQTT will be used as communication channel between the webapp and the backend. MQTT broker is "mqtt.mydomain.com". I didn't run the broker yet. +I already setup the project structure. Can you implement the app? +To test whether this Dioxus project can be build, you may use this command "dx bundle --web --release --debug-symbols=false" + +P.S. In a Dioxus single-page application (SPA), switching screens can be handled perfectly using standard Rust state matching (often called conditional rendering or state-based routing). + diff --git a/Cargo.toml b/Cargo.toml index b675ca2..580abd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,9 @@ futures = "0.3" tempfile = "3" [[example]] -name = "smartsend_example" -path = "examples/smartsend_example.rs" +name = "smartpack_example" +path = "examples/smartpack_example.rs" [[example]] -name = "smartreceive_example" -path = "examples/smartreceive_example.rs" +name = "smartunpack_example" +path = "examples/smartunpack_example.rs" diff --git a/README.md b/README.md index fc6825b..6590d76 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ msghandler enables seamless communication across multiple platforms through NATS using msghandler data = [("message", "Hello World", "text")] -env, env_json_str = smartsend("/chat/room1", data; broker_url="nats://localhost:4222") +env, env_json_str = smartpack("/chat/room1", data; broker_url="nats://localhost:4222") println("Message sent!") ``` @@ -120,7 +120,7 @@ println("Message sent!") import msghandler from './src/msghandler_ssr.js'; const data = [["message", "Hello World", "text"]]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { broker_url: "nats://localhost:4222" } @@ -134,7 +134,7 @@ console.log("Message sent!"); import msghandler from './src/msghandler_csr.js'; const data = [["message", "Hello World", "text"]]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { broker_url: "ws://localhost:4222" } @@ -145,10 +145,10 @@ console.log("Message sent!"); #### Python ```python -from msghandler import smartsend +from msghandler import smartpack data = [("message", "Hello World", "text")] -env, env_json_str = await smartsend( +env, env_json_str = await smartpack( "/chat/room1", data, broker_url="nats://localhost:4222" @@ -159,10 +159,10 @@ print("Message sent!") #### MicroPython ```python -from msghandler import smartsend +from msghandler import smartpack data = [("message", "Hello World", "text")] -env, env_json_str = smartsend( +env, env_json_str = smartpack( "/chat/room1", data, broker_url="nats://localhost:4222", @@ -179,12 +179,12 @@ print("Message sent!") All platforms use the same input/output format for payloads: -**Input format for `smartsend`:** +**Input format for `smartpack`:** ``` [(dataname1, data1, type1), (dataname2, data2, type2), ...] ``` -**Output format for `smartreceive`:** +**Output format for `smartunpack`:** ```json { "correlation_id": "...", @@ -204,7 +204,7 @@ All platforms use the same input/output format for payloads: } ``` -### smartsend +### smartpack Sends data either directly via NATS or via a fileserver URL, depending on payload size. @@ -213,7 +213,7 @@ Sends data either directly via NATS or via a fileserver URL, depending on payloa ```julia using msghandler -env, env_json_str = msghandler.smartsend( +env, env_json_str = msghandler.smartpack( subject::String, data::AbstractArray{Tuple{String, Any, String}}; broker_url::String = "nats://localhost:4222", @@ -240,7 +240,7 @@ env, env_json_str = msghandler.smartsend( ```javascript import msghandler from './src/msghandler_ssr.js'; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( subject, data, // Array of [dataname, data, type] tuples { @@ -269,7 +269,7 @@ const [env, env_json_str] = await msghandler.smartsend( ```javascript import msghandler from './src/msghandler_csr.js'; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( subject, data, { @@ -298,7 +298,7 @@ const [env, env_json_str] = await msghandler.smartsend( ```python from msghandler import msghandler -env, env_json_str = await msghandler.smartsend( +env, env_json_str = await msghandler.smartpack( subject: str, data: List[Tuple[str, Any, str]], broker_url: str = "nats://localhost:4222", @@ -326,7 +326,7 @@ env, env_json_str = await msghandler.smartsend( from msghandler import msghandler # Limited to direct transport (< 100KB threshold) -env, env_json_str = msghandler.smartsend( +env, env_json_str = msghandler.smartpack( subject, data, # List of (dataname, data, type) tuples broker_url="nats://localhost:4222", @@ -335,7 +335,7 @@ env, env_json_str = msghandler.smartsend( # Returns: Tuple[Dict, str] ``` -### smartreceive +### smartunpack Receives and processes messages from NATS, handling both direct and link transport. @@ -344,7 +344,7 @@ Receives and processes messages from NATS, handling both direct and link transpo ```julia using msghandler -env = msghandler.smartreceive( +env = msghandler.smartunpack( msg::NATS.Msg; fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -359,7 +359,7 @@ env = msghandler.smartreceive( ```javascript import msghandler from './src/msghandler_ssr.js'; -const env = await msghandler.smartreceive( +const env = await msghandler.smartunpack( msg, { fileserver_download_handler: msghandler.fetchWithBackoff, @@ -376,7 +376,7 @@ const env = await msghandler.smartreceive( ```javascript import msghandler from './src/msghandler_csr.js'; -const env = await msghandler.smartreceive( +const env = await msghandler.smartunpack( msg, { fileserver_download_handler: msghandler.fetchWithBackoff, @@ -393,7 +393,7 @@ const env = await msghandler.smartreceive( ```python from msghandler import msghandler -env = await msghandler.smartreceive( +env = await msghandler.smartunpack( msg, fileserver_download_handler=fetch_with_backoff, max_retries=5, @@ -408,7 +408,7 @@ env = await msghandler.smartreceive( ```python from msghandler import msghandler -env = msghandler.smartreceive( +env = msghandler.smartunpack( msg, fileserver_download_handler=_sync_fileserver_download, max_retries=3, @@ -452,7 +452,7 @@ data = [ ("large_document", large_file_data, "binary") ] -env, env_json_str = smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") +env, env_json_str = smartpack("/chat/room1", data; fileserver_url="http://localhost:8080") ``` #### JavaScript (Node.js) @@ -466,7 +466,7 @@ const data = [ ["large_document", largeFileData, "binary"] ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { fileserver_url: 'http://localhost:8080' } @@ -484,7 +484,7 @@ const data = [ ["large_document", largeFileData, "binary"] ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/chat/room1", data, { broker_url: 'ws://localhost:4222', fileserver_url: 'http://localhost:8080' } @@ -502,7 +502,7 @@ data = [ ("large_document", large_file_data, "binary") ] -env, env_json_str = await msghandler.smartsend( +env, env_json_str = await msghandler.smartpack( "/chat/room1", data, fileserver_url="http://localhost:8080" @@ -525,7 +525,7 @@ config = Dict( ) data = [("config", config, "dictionary")] -env, env_json_str = smartsend("/device/config", data) +env, env_json_str = smartpack("/device/config", data) ``` #### JavaScript (Node.js) @@ -539,7 +539,7 @@ const config = { update_interval: 60 }; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/device/config", [["config", config, "dictionary"]] ); @@ -557,7 +557,7 @@ config = { } data = [("config", config, "dictionary")] -env, env_json_str = await msghandler.smartsend("/device/config", data) +env, env_json_str = await msghandler.smartpack("/device/config", data) ``` ### Example 3: Table Data (Arrow IPC) @@ -577,7 +577,7 @@ df = DataFrame( ) data = [("students", df, "arrowtable")] -env, env_json_str = smartsend("/data/analysis", data) +env, env_json_str = smartpack("/data/analysis", data) ``` #### JavaScript (Node.js) @@ -591,7 +591,7 @@ const df = [ { id: 3, name: "Charlie", score: 92 } ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/data/analysis", [["students", df, "arrowtable"]] ); @@ -610,7 +610,7 @@ df = pd.DataFrame({ }) data = [("students", df, "arrowtable")] -env, env_json_str = await msghandler.smartsend("/data/analysis", data) +env, env_json_str = await msghandler.smartpack("/data/analysis", data) ``` #### JavaScript (Browser) @@ -626,7 +626,7 @@ const df = [ { id: 3, name: "Charlie", score: 92 } ]; -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/data/analysis", [["students", df, "jsontable"]], // Use jsontable for browser { broker_url: 'ws://localhost:4222' } @@ -643,7 +643,7 @@ Bi-directional communication with reply-to support. using msghandler # Requester -env, env_json_str = smartsend( +env, env_json_str = smartpack( "/device/command", [("command", Dict("action" => "read_sensor"), "dictionary")]; broker_url="nats://localhost:4222", @@ -652,9 +652,9 @@ env, env_json_str = smartsend( # Receiver (in separate application) msg = NATS.subscription.next() -env = smartreceive(msg) +env = smartunpack(msg) # Process request and send response -response_env, response_json = smartsend( +response_env, response_json = smartpack( "/device/response", [("result", Dict("value" => 42), "dictionary")], reply_to="/device/command", @@ -668,7 +668,7 @@ response_env, response_json = smartsend( import msghandler from './src/msghandler_ssr.js'; // Requester -const [env, env_json_str] = await msghandler.smartsend( +const [env, env_json_str] = await msghandler.smartpack( "/device/command", [["command", { action: "read_sensor" }, "dictionary"]], { broker_url: 'nats://localhost:4222', reply_to: '/device/response' } @@ -676,9 +676,9 @@ const [env, env_json_str] = await msghandler.smartsend( // Receiver (in separate application) // const msg = await natsConsumer.next(); -// const env = await msghandler.smartreceive(msg); +// const env = await msghandler.smartunpack(msg); // Process request and send response -// const response_env, response_json = await msghandler.smartsend( +// const response_env, response_json = await msghandler.smartpack( // "/device/response", // [["result", { value: 42 }, "dictionary"]], // { reply_to: '/device/command', reply_to_msg_id: env.msg_id } @@ -691,7 +691,7 @@ const [env, env_json_str] = await msghandler.smartsend( from msghandler import msghandler # Requester -env, env_json_str = await msghandler.smartsend( +env, env_json_str = await msghandler.smartpack( "/device/command", [("command", {"action": "read_sensor"}, "dictionary")], broker_url="nats://localhost:4222", @@ -700,9 +700,9 @@ env, env_json_str = await msghandler.smartsend( # Receiver (in separate application) # msg = await nats_consumer.next() -# env = await msghandler.smartreceive(msg) +# env = await msghandler.smartunpack(msg) # Process request and send response -# response_env, response_json = await msghandler.smartsend( +# response_env, response_json = await msghandler.smartpack( # "/device/response", # [("result", {"value": 42}, "dictionary")], # reply_to="/device/command", @@ -895,7 +895,7 @@ node build.js import msghandlerCSR from './dist/msghandler-csr-bundle.js'; // Use the library - const [env, envJson] = await msghandlerCSR.smartsend( + const [env, envJson] = await msghandlerCSR.smartpack( "/chat/user/v1/message", [["msg", "Hello", "text"]], { broker_url: "wss://nats.example.com" } diff --git a/docs/architecture.md b/docs/architecture.md index 458677f..1259106 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -130,8 +130,8 @@ flowchart TD ```mermaid flowchart TD subgraph "msghandler Module" - SmartSend[smartsend Function] - SmartReceive[smartreceive Function] + smartpack[smartpack Function] + smartunpack[smartunpack Function] Serialize[_serialize_data] Deserialize[_deserialize_data] @@ -149,18 +149,18 @@ flowchart TD Envelope[msg_envelope_v1 Struct] end - SmartSend --> Serialize - SmartSend --> EnvelopeToJson - SmartSend --> FileServerUpload + smartpack --> Serialize + smartpack --> EnvelopeToJson + smartpack --> FileServerUpload - SmartReceive --> Deserialize - SmartReceive --> FileServerDownload + smartunpack --> Deserialize + smartunpack --> FileServerDownload EnvelopeToJson --> Envelope Serialize --> Payload - style SmartSend fill:#d1fae5,stroke:#10b981 - style SmartReceive fill:#d1fae5,stroke:#10b981 + style smartpack fill:#d1fae5,stroke:#10b981 + style smartunpack fill:#d1fae5,stroke:#10b981 style FileServerUpload fill:#fef3c7,stroke:#f59e0b style FileServerDownload fill:#fef3c7,stroke:#f59e0b ``` @@ -173,8 +173,8 @@ flowchart TD | Component | Purpose | Platform Support | |-----------|---------|------------------| -| **smartsend** | Send data with automatic transport selection, returns (envelope, json_string) for caller to publish via transport | All | -| **smartreceive** | Receive and process messages from JSON string | All | +| **smartpack** | Send data with automatic transport selection, returns (envelope, json_string) for caller to publish via transport | All | +| **smartunpack** | Receive and process messages from JSON string | All | | **_serialize_data** | Serialize data according to payload type | All | | **_deserialize_data** | Deserialize bytes to native data types | All | | **envelope_to_json** | Convert msg_envelope_v1 struct to JSON string | All | @@ -187,7 +187,7 @@ flowchart TD ```mermaid flowchart TD - A[User calls smartsend subject data] --> B[Process each payload] + A[User calls smartpack subject data] --> B[Process each payload] B --> C{Calculate serialized size} C -->|Size < Threshold| D[Direct Transport] C -->|Size >= Threshold| E[Link Transport] @@ -349,7 +349,7 @@ flowchart TD ```mermaid flowchart TD - A[smartsend called] --> B[Serialize payload] + A[smartpack called] --> B[Serialize payload] B --> C[Calculate size] C --> D{Size < Threshold?} @@ -560,7 +560,7 @@ pub enum Payload { } // Configuration via builder pattern -pub struct SmartsendOptions { +pub struct smartpackOptions { pub broker_url: String, pub fileserver_url: String, pub fileserver_upload_handler: Option>, @@ -577,7 +577,7 @@ let conn = transport_client::connect(DEFAULT_BROKER_URL).await?; // Subscribe and process messages let mut sub = conn.subscribe("/agent/wine/api/v1/analyze")?; for msg in sub.messages() { - let envelope = smartreceive(&String::from_utf8_lossy(&msg.payload), &Default::default()).await?; + let envelope = smartunpack(&String::from_utf8_lossy(&msg.payload), &Default::default()).await?; // Access deserialized payloads by type for payload in &envelope.payloads { match payload.payload_type.as_str() { @@ -840,11 +840,11 @@ flowchart TD | - | - | Updated diagrams to use generic "Message Broker" instead of "NATS Server" | All sections | | - | - | Updated code examples to use transport-agnostic patterns | All sections | | - | - | Removed NATS client packages from external dependencies | All sections | -| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartreceive` deserialization changes | All sections | -| - | - | `smartreceive` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | +| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | +| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | | - | - | Added `plik_upload_file` convenience function to component table | specification.md:13 | | - | - | Fixed Rust payload access pattern (data is String, not Payload enum) | All sections | -| - | - | Fixed `SmartsendOptions.fileserver_upload_handler` type to `Arc` | specification.md:13 | +| - | - | Fixed `smartpackOptions.fileserver_upload_handler` type to `Arc` | specification.md:13 | | - | - | Removed `metadata` from link transport examples (now `None`/omitted) | specification.md:3 | | - | - | Removed duplicate footer text | All sections | | 2026-05-13 | 1.3.0 | Added Rust support with tokio, serde, and arrow2 | All sections | @@ -854,9 +854,9 @@ flowchart TD | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | | - | - | Removed publish_message component (commented out in source) | | - | - | Removed NATSClient and NATSConnectionPool classes (not in ground truth) | -| - | - | Updated smartsend to return JSON for caller to publish via transport | +| - | - | Updated smartpack to return JSON for caller to publish via transport | | - | - | Updated component diagram to match actual module structure | -| - | - | Updated data flow to show smartsend returns JSON for caller to publish | +| - | - | Updated data flow to show smartpack returns JSON for caller to publish | | - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | | 2026-03-15 | 1.1.0 | JavaScript connection management | | - | - | Added NATSClient with keepAlive support | diff --git a/docs/requirements.md b/docs/requirements.md index fc20b68..ce93f63 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -54,7 +54,7 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless |---------|-------------| | Cross-platform interoperability | Seamless data exchange between Julia, JavaScript, Python, Dart, Rust, and MicroPython | | Intelligent transport selection | Direct transport (<0.5MB) vs Link transport (≥0.5MB) based on payload size | -| Unified API | Consistent `smartsend()` and `smartreceive()` functions across all platforms | +| Unified API | Consistent `smartpack()` and `smartunpack()` functions across all platforms | | Multi-payload support | List of (dataname, data, type) tuples with appropriate handling | | File server integration | Plik one-shot upload and custom HTTP server support | | Reliability features | Exponential backoff retry and correlation ID propagation | @@ -323,10 +323,10 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless ## 11. API Contract -### 11.1 smartsend Signature +### 11.1 smartpack Signature ```julia -function smartsend( +function smartpack( subject::String, data::AbstractArray{Tuple{String, T1, String}, 1}; broker_url::String = DEFAULT_BROKER_URL, @@ -345,12 +345,12 @@ function smartsend( )::Tuple{msg_envelope_v1, String} where {T1<:Any} ``` -**Note**: Publishing via the transport layer is the caller's responsibility. `smartsend` returns `(env::msg_envelope_v1, env_json_str::String)`. +**Note**: Publishing via the transport layer is the caller's responsibility. `smartpack` returns `(env::msg_envelope_v1, env_json_str::String)`. -### 11.2 smartreceive Signature +### 11.2 smartunpack Signature ```julia -function smartreceive( +function smartunpack( msg_json_str::String; fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -359,9 +359,9 @@ function smartreceive( )::JSON.Object{String, Any} ``` -**Note**: Pass the payload string from the transport subscription to `smartreceive`. The input is the JSON string payload from the transport message, not the transport message object directly. +**Note**: Pass the payload string from the transport subscription to `smartunpack`. The input is the JSON string payload from the transport message, not the transport message object directly. -**Note**: Pass the payload from the transport subscription to `smartreceive`. +**Note**: Pass the payload from the transport subscription to `smartunpack`. --- @@ -411,8 +411,8 @@ function smartreceive( | - | - | Updated all NATS references to generic "transport layer"/"message broker" | | - | - | Removed NATS client packages from dependencies tables | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | -| - | - | Fixed smartsend signature: removed is_publish, NATS_connection; added sender_name | -| - | - | Fixed smartreceive signature: takes msg_json_str::String instead of msg::NATS.Msg | +| - | - | Fixed smartpack signature: removed is_publish, NATS_connection; added sender_name | +| - | - | Fixed smartunpack signature: takes msg_json_str::String instead of msg::NATS.Msg | | - | - | Fixed size_threshold default from 1,000,000 to 500,000 | | - | - | Updated FR-013/FR-014 to reflect caller responsibility for NATS publishing | | - | - | Updated FR-008/FR-009 to include file path upload overload | diff --git a/docs/specification.md b/docs/specification.md index 2b32c91..df773f6 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -13,8 +13,8 @@ This document defines the **technical contract** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. This specification serves as the single source of truth for: -- **Inputs**: What data structures are accepted by `smartsend()` -- **Outputs**: What data structures are returned by `smartreceive()` +- **Inputs**: What data structures are accepted by `smartpack()` +- **Outputs**: What data structures are returned by `smartunpack()` - **Data Shapes**: Exact field names, types, and constraints - **Error Codes**: Standardized error responses for failure scenarios @@ -24,7 +24,7 @@ This specification serves as the single source of truth for: |----------------------|-------------------|-------------| | Section 2 (Message Envelope) | FR-012, FR-013, NFR-101, NFR-102 | Message envelope structure and validation | | Section 3 (Payload Schema) | FR-001, FR-002, FR-003, FR-004, NFR-101, NFR-102 | Payload structure and field definitions | -| Section 4 (Payload Format) | FR-006, FR-007 | Tuple format for smartsend() | +| Section 4 (Payload Format) | FR-006, FR-007 | Tuple format for smartpack() | | Section 5 (Enumerations) | FR-003, FR-004, FR-006, NFR-101 | Enumerations for transport and encoding | | Section 6 (Transport Protocols) | FR-003, FR-004, NFR-104, NFR-105 | Direct and link transport protocols | | Section 7 (Size Thresholds) | FR-004, FR-005, NFR-104, NFR-105 | Size thresholds for transport selection | @@ -143,9 +143,9 @@ This specification serves as the single source of truth for: ## Payload Format -### Tuple Format for `smartsend()` +### Tuple Format for `smartpack()` -The `smartsend()` function accepts data as an array of tuples with the format: +The `smartpack()` function accepts data as an array of tuples with the format: ``` ("data_name", data, "data_type") @@ -161,17 +161,17 @@ The `smartsend()` function accepts data as an array of tuples with the format: ```julia # Julia -smartsend("/chat/user/v1/message", [("msg", "Hello World", "text")]) +smartpack("/chat/user/v1/message", [("msg", "Hello World", "text")]) ``` ```python # Python -await smartsend("/chat/user/v1/message", [("msg", "Hello World", "text")]) +await smartpack("/chat/user/v1/message", [("msg", "Hello World", "text")]) ``` ```typescript // JavaScript -await smartsend("/chat/user/v1/message", [["msg", "Hello World", "text"]]); +await smartpack("/chat/user/v1/message", [["msg", "Hello World", "text"]]); ``` ### Multiple Payloads Example @@ -182,7 +182,7 @@ data = [ ("msg", "Hello", "text"), ("img", binary_data, "image") ] -smartsend("/agent/v1/process", data) +smartpack("/agent/v1/process", data) ``` ```python @@ -191,7 +191,7 @@ data = [ ("msg", "Hello", "text"), ("img", binary_data, "image") ] -await smartsend("/agent/v1/process", data) +await smartpack("/agent/v1/process", data) ``` ### Data Type Mapping @@ -411,12 +411,12 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa ## API Contract -### `smartsend` Function Signature +### `smartpack` Function Signature #### Julia ```julia -function smartsend( +function smartpack( subject::String, data::AbstractArray{Tuple{String, T1, String}, 1}; broker_url::String = DEFAULT_BROKER_URL, @@ -440,7 +440,7 @@ function smartsend( #### Python ```python -async def smartsend( +async def smartpack( subject: str, data: List[Tuple[str, Any, str]], broker_url: str = DEFAULT_BROKER_URL, @@ -464,7 +464,7 @@ async def smartsend( #### JavaScript (Node.js) ```typescript -async function smartsend( +async function smartpack( subject: string, data: Array<[string, any, string]>, options?: { @@ -490,7 +490,7 @@ async function smartsend( #### JavaScript (Browser) ```typescript -async function smartsend( +async function smartpack( subject: string, data: Array<[string, any, string]>, options?: { @@ -516,7 +516,7 @@ async function smartsend( #### MicroPython ```python -def smartsend( +def smartpack( subject: str, data: List[Tuple[str, Any, str]], size_threshold: int = 100_000, # Lower threshold for memory constraints @@ -529,7 +529,7 @@ def smartsend( #### Dart (Desktop/Flutter) ```dart -Future<[Map, String]> smartsend( +Future<[Map, String]> smartpack( String subject, List> data, { String brokerUrl = DEFAULT_BROKER_URL, @@ -553,7 +553,7 @@ Future<[Map, String]> smartsend( #### Dart Web ```dart -Future<[Map, String]> smartsend( +Future<[Map, String]> smartpack( String subject, List> data, { String brokerUrl = DEFAULT_BROKER_URL, @@ -578,14 +578,14 @@ Future<[Map, String]> smartsend( #### Rust ```rust -pub async fn smartsend( +pub async fn smartpack( subject: &str, data: &[(String, Payload, String)], - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), msghandlerError> -// SmartsendOptions struct -pub struct SmartsendOptions { +// smartpackOptions struct +pub struct smartpackOptions { pub broker_url: String, pub fileserver_url: String, pub fileserver_upload_handler: Option, @@ -636,12 +636,12 @@ pub struct MsgEnvelopeV1 { **Note**: Publishing via the transport layer is the caller's responsibility. Returns `Result<(MsgEnvelopeV1, String), msghandlerError>`. Uses `serde` for JSON serialization. -### `smartreceive` Function Signature +### `smartunpack` Function Signature #### Julia ```julia -function smartreceive( +function smartunpack( msg_json_str::String; # Pass payload from transport subscription fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, @@ -655,7 +655,7 @@ function smartreceive( #### Python ```python -async def smartreceive( +async def smartunpack( msg_json_str: str, # JSON string from transport message payload fileserver_download_handler: Callable = fetch_with_backoff, max_retries: int = 5, @@ -669,7 +669,7 @@ async def smartreceive( #### JavaScript (Node.js) ```typescript -async function smartreceive( +async function smartunpack( msg_json_str: string, // JSON string from transport message payload options?: { fileserver_download_handler?: Function; @@ -683,7 +683,7 @@ async function smartreceive( #### JavaScript (Browser) ```typescript -async function smartreceive( +async function smartunpack( msg_json_str: string, // JSON string from transport message payload options?: { fileserver_download_handler?: Function; @@ -699,7 +699,7 @@ async function smartreceive( #### MicroPython ```python -def smartreceive(msg_json_str: str, **kwargs) -> Dict[str, Any]: +def smartunpack(msg_json_str: str, **kwargs) -> Dict[str, Any]: ``` **Note**: Input is the JSON string payload from the transport message. @@ -707,7 +707,7 @@ def smartreceive(msg_json_str: str, **kwargs) -> Dict[str, Any]: #### Dart (Desktop/Flutter) ```dart -Future> smartreceive( +Future> smartunpack( Map msg_json_str, // JSON object from transport message payload { Function? fileserverDownloadHandler, @@ -722,7 +722,7 @@ Future> smartreceive( #### Dart Web ```dart -Future> smartreceive( +Future> smartunpack( Map msg_json_str, // JSON object from transport message payload { Function? fileserverDownloadHandler, @@ -737,13 +737,13 @@ Future> smartreceive( #### Rust ```rust -pub async fn smartreceive( +pub async fn smartunpack( msg_json_str: &str, // JSON string from transport message payload - options: &SmartreceiveOptions, + options: &smartunpackOptions, ) -> Result -// SmartreceiveOptions struct -pub struct SmartreceiveOptions { +// smartunpackOptions struct +pub struct smartunpackOptions { pub fileserver_download_handler: Option, pub max_retries: u32, pub base_delay: u64, @@ -933,7 +933,7 @@ The browser implementation ([`src/msghandler_csr.js`](../src/msghandler_csr.js)) ```mermaid flowchart TD - A[User calls smartsend subject data] --> B[Serialize payload according to payload_type] + A[User calls smartpack subject data] --> B[Serialize payload according to payload_type] B --> C{Calculate serialized size} C -->|Size < Threshold| D[Direct Transport: Encode as Base64] C -->|Size >= Threshold| E[Link Transport: Upload to file server] @@ -1156,8 +1156,8 @@ flowchart TD | - | - | Updated all NATS references to generic "transport layer"/"message broker" | All | | - | - | Removed NATS client packages from dependencies tables | All | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All | -| - | - | Updated smartsend signatures: removed is_publish, nats_connection; added sender_name | FR-001 through FR-014 | -| - | - | Updated smartreceive signatures: takes msg_json_str::String instead of msg | FR-001 through FR-014 | +| - | - | Updated smartpack signatures: removed is_publish, nats_connection; added sender_name | FR-001 through FR-014 | +| - | - | Updated smartunpack signatures: takes msg_json_str::String instead of msg | FR-001 through FR-014 | | - | - | Removed publishMessage function and NATSClient/NATSConnectionPool classes from browser section | FR-013, FR-014 | | - | - | Added plik_oneshot_upload(filepath) overload to file server interface | FR-008, FR-009 | | - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | FR-003, FR-004 | diff --git a/docs/walkthrough.md b/docs/walkthrough.md index a489997..9ee50c2 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -44,7 +44,7 @@ flowchart TB subgraph msghandler["msghandler Module"] direction TB - subgraph Sender["Sender (smartsend)"] + subgraph Sender["Sender (smartpack)"] direction LR S1["Data Tuples
[(dataname, data, type)]"] S2["Serialize Data"] @@ -60,7 +60,7 @@ flowchart TB S5 --> S6 end - subgraph Receiver["Receiver (smartreceive)"] + subgraph Receiver["Receiver (smartunpack)"] direction LR R1["Subscribe via transport"] R2["Parse Envelope"] @@ -101,7 +101,7 @@ flowchart TB |-----------|-------------|-----------| | **Claim-Check Pattern** | Large payloads uploaded to HTTP server, URL sent via transport | Transport has message size limits; avoids overflow | | **Automatic Transport Selection** | Direct (< threshold) vs Link (≥ threshold) based on size | Optimizes memory vs network I/O trade-off | -| **Cross-Platform API** | Consistent `smartsend()`/`smartreceive()` across all platforms | Simplifies developer experience | +| **Cross-Platform API** | Consistent `smartpack()`/`smartunpack()` across all platforms | Simplifies developer experience | | **Exponential Backoff** | Retry downloads with increasing delays | Handles transient failures gracefully | --- @@ -118,7 +118,7 @@ A JavaScript chat webapp wants to send mixed payloads (text message + user avata ```javascript // JavaScript (Browser or Node.js) -const [env, msgJson] = await msghandler.smartsend( +const [env, msgJson] = await msghandler.smartpack( "/agent/wine/api/v1/prompt", [ ["msg", "Hello! I'm Ton.", "text"], @@ -225,14 +225,14 @@ msghandler builds the message envelope: **Rationale**: - The transport layer provides message delivery (NATS, MQTT, WebSocket, etc.) - JSON format ensures cross-platform compatibility -- `smartsend()` returns `(env, msgJson)` - caller handles publishing via their chosen transport +- `smartpack()` returns `(env, msgJson)` - caller handles publishing via their chosen transport #### Step 6: Julia Backend Receives Message ```julia # Julia backend transport_msg = transport_subscription.next() # Get message from transport -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # env["payloads"] is now: # [ @@ -242,7 +242,7 @@ env = smartreceive(String(transport_msg.payload)) ``` **Rationale**: -- `smartreceive()` handles both transport types automatically +- `smartunpack()` handles both transport types automatically - Deserialization is type-aware based on `payload_type` - Returns consistent tuple format regardless of transport @@ -253,7 +253,7 @@ env = smartreceive(String(transport_msg.payload)) response_text = "Hello Ton! I'm the AI assistant." generated_image = generate_ai_image(response_text) -env, msg_json = smartsend( +env, msg_json = smartpack( "/agent/wine/api/v1/response", [ ("response", response_text, "text"), @@ -282,7 +282,7 @@ A JavaScript webapp wants to upload a large file (10MB) to a Julia backend for p #### Step 1: JavaScript Webapp Sends Large File ```javascript -const [env, msgJson] = await msghandler.smartsend( +const [env, msgJson] = await msghandler.smartpack( "/agent/wine/api/v1/process", [ ["file", largeFileData, "binary"] @@ -358,7 +358,7 @@ const response = await plikOneshotUpload( ```julia # Julia backend transport_msg = transport_subscription.next() -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # msghandler automatically: # 1. Extracts URL from payload @@ -386,7 +386,7 @@ A Python application sends tabular data (pandas DataFrame) to a Julia backend fo ```python # Python import pandas as pd -from msghandler import smartsend +from msghandler import smartpack df = pd.DataFrame({ "id": [1, 2, 3], @@ -394,7 +394,7 @@ df = pd.DataFrame({ "score": [95, 88, 92] }) -env, msg_json = await smartsend( +env, msg_json = await smartpack( "/agent/wine/api/v1/analyze", [("data", df, "arrowtable")], broker_url=DEFAULT_BROKER_URL, @@ -431,7 +431,7 @@ arrow_bytes = buf.getvalue() ```julia # Julia backend transport_msg = transport_subscription.next() -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # env["payloads"][1] is now: # ("data", DataFrame with id, name, score columns, "arrowtable") @@ -449,7 +449,7 @@ env = smartreceive(String(transport_msg.payload)) results = analyze_data(env["payloads"][1][2]) # Send results back -env, msg_json = smartsend( +env, msg_json = smartpack( "/agent/wine/api/v1/results", [("results", results, "arrowtable")], reply_to = "/python/worker/v1/results" @@ -475,7 +475,7 @@ A Rust service needs to process messages from a Julia analytics pipeline and sen ```rust // Rust service - using tokio async runtime -use msghandler::{smartreceive, MsgEnvelopeV1}; +use msghandler::{smartunpack, MsgEnvelopeV1}; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; #[tokio::main] @@ -486,7 +486,7 @@ async fn main() { let mut sub = conn.subscribe("/agent/wine/api/v1/analyze").unwrap(); for msg in sub.messages() { - let envelope = smartreceive( + let envelope = smartunpack( &String::from_utf8_lossy(&msg.payload), &Default::default(), ).await.unwrap(); @@ -495,7 +495,7 @@ async fn main() { for payload in &envelope.payloads { match payload.payload_type.as_str() { "arrowtable" => { - // Data is base64-encoded Arrow IPC bytes after smartreceive() + // Data is base64-encoded Arrow IPC bytes after smartunpack() let arrow_bytes = BASE64.decode(&payload.data).unwrap(); println!("Received arrowtable payload ({} bytes)", arrow_bytes.len()); }, @@ -522,19 +522,19 @@ async fn main() { **Rationale**: - **serde serialization**: Automatic JSON deserialization to `MsgEnvelopeV1` - **tokio runtime**: Efficient async I/O for transport and HTTP operations -- **smartreceive deserialization**: Payload data is deserialized and stored as strings in `payload.data` +- **smartunpack deserialization**: Payload data is deserialized and stored as strings in `payload.data` - **Type dispatch**: `payload_type` field determines how to interpret the `data` string #### Step 2: Rust Service Sends Processed Results ```rust // Rust service sends results back with mixed payload types -use msghandler::{smartsend, Payload, SmartsendOptions}; +use msghandler::{smartpack, Payload, smartpackOptions}; let results_df = /* processed Arrow table */; let result_bytes = /* serialize to Arrow IPC */; -let (envelope, json_str) = smartsend( +let (envelope, json_str) = smartpack( "/agent/wine/api/v1/results", &[ ( @@ -548,7 +548,7 @@ let (envelope, json_str) = smartsend( "text".to_string(), ), ], - &SmartsendOptions { + &smartpackOptions { broker_url: DEFAULT_BROKER_URL.to_string(), reply_to: "/python/worker/v1/results".to_string(), msg_purpose: "chat".to_string(), @@ -561,7 +561,7 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; ``` **Rationale**: -- **Builder pattern**: `SmartsendOptions` provides clean configuration +- **Builder pattern**: `smartpackOptions` provides clean configuration - **Enum-based payloads**: Type safety prevents sending incorrect data types - **Default options**: sensible defaults reduce boilerplate - **Result**: idiomatic Rust error handling @@ -570,7 +570,7 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; ```python # Python backend receives Rust response -env = await smartreceive(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)) # env["payloads"][0] is now: # ("results", arrow_table_data, "arrowtable") @@ -589,7 +589,7 @@ env = await smartreceive(str(transport_msg.payload)) // Rust service sends large binary file via link transport let large_file_data: Vec = std::fs::read("/data/large_dataset.parquet")?; -let (envelope, json_str) = smartsend( +let (envelope, json_str) = smartpack( "/agent/wine/api/v1/upload", &[ ( @@ -598,7 +598,7 @@ let (envelope, json_str) = smartsend( "binary".to_string(), ), ], - &SmartsendOptions { + &smartpackOptions { broker_url: DEFAULT_BROKER_URL.to_string(), fileserver_url: DEFAULT_FILESERVER_URL.to_string(), size_threshold: DEFAULT_SIZE_THRESHOLD, // threshold triggers link transport @@ -627,7 +627,7 @@ A MicroPython sensor device sends sensor readings to a Python backend. ```python # MicroPython -from msghandler import smartsend +from msghandler import smartpack sensor_data = { "temperature": 25.5, @@ -635,7 +635,7 @@ sensor_data = { "pressure": 1013.25 } -env, msg_json = smartsend( +env, msg_json = smartpack( "/sensor/device/v1/readings", [("data", sensor_data, "dictionary")], broker_url=DEFAULT_BROKER_URL, @@ -667,7 +667,7 @@ payload_b64 = base64.b64encode(json_bytes).decode('ascii') ```python # Python backend transport_msg = await transport_consumer.next() -env = await smartreceive(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)) # env["payloads"][0] is now: # ("data", {"temperature": 25.5, "humidity": 60.0, ...}, "dictionary") @@ -692,7 +692,7 @@ Multiple platforms (JavaScript, Python, Julia) communicate in a chat application ```javascript // JavaScript (Frontend) -const [env, msgJson] = await msghandler.smartsend( +const [env, msgJson] = await msghandler.smartpack( "/chat/user/v1/message", [ ["text", "Check this out!", "text"], @@ -716,7 +716,7 @@ const [env, msgJson] = await msghandler.smartsend( ```python # Python (Backend) transport_msg = await transport_consumer.next() -env = await smartreceive(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)) # env["payloads"] is now: # [ @@ -735,7 +735,7 @@ env = await smartreceive(str(transport_msg.payload)) ```julia # Julia (Backend) transport_msg = transport_subscription.next() -env = smartreceive(String(transport_msg.payload)) +env = smartunpack(String(transport_msg.payload)) # env["payloads"] is now: # [ @@ -755,7 +755,7 @@ Each platform can reply using the same API: ```python # Python reply -await smartsend( +await smartpack( "/chat/user/v1/reply", [("response", "Nice!", "text")], reply_to="/chat/user/v1/message" @@ -764,7 +764,7 @@ await smartsend( ```julia # Julia reply -smartsend( +smartpack( "/chat/user/v1/reply", [("response", "Nice!", "text")], reply_to="/chat/user/v1/message" @@ -773,7 +773,7 @@ smartsend( ```javascript // JavaScript reply -await msghandler.smartsend( +await msghandler.smartpack( "/chat/user/v1/reply", [["response", "Nice!", "text"]], { reply_to: "/chat/user/v1/message" } @@ -827,14 +827,14 @@ Every message includes a `correlation_id`: correlation_id = string(uuid4()) # Use throughout the flow -log_trace(correlation_id, "Starting smartsend") +log_trace(correlation_id, "Starting smartpack") log_trace(correlation_id, "Serialized payload size: 100 bytes") log_trace(correlation_id, "Published to transport") ``` **Log Format**: ``` -[2026-03-13T16:30:00.000Z] [Correlation: abc123...] Starting smartsend +[2026-03-13T16:30:00.000Z] [Correlation: abc123...] Starting smartpack [2026-03-13T16:30:00.001Z] [Correlation: abc123...] Serialized payload size: 100 bytes [2026-03-13T16:30:00.002Z] [Correlation: abc123...] Published to transport ``` @@ -928,8 +928,8 @@ log_trace(correlation_id, "Published to transport") | - | - | Removed all NATS-specific references from walkthrough | All sections | | - | - | Updated code examples to use transport-agnostic patterns | All sections | | - | - | Updated diagrams to remove NATS-specific labels | All sections | -| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartreceive` deserialization changes | All sections | -| - | - | `smartreceive` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | +| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | +| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | | - | - | Added `plik_upload_file` convenience function documentation | specification.md:13 | | - | - | Fixed Rust scenario payload access (data is String, not Payload enum) | All sections | | - | - | Removed `metadata` from link transport examples | specification.md:3 | @@ -937,7 +937,7 @@ log_trace(correlation_id, "Published to transport") | - | - | Added Rust user scenario (User Scenario 4) | specification.md:11 (Rust API) | | - | - | Updated scenario numbering (MicroPython → Scenario 5, Cross-Platform → Scenario 6) | All sections | | 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All sections | -| - | - | Updated smartreceive calls to use transport payload pattern | All sections | +| - | - | Updated smartunpack calls to use transport payload pattern | All sections | | - | - | Removed NATSClient.publish() calls (caller responsible for transport publishing) | All sections | | - | - | Removed is_publish and nats_connection parameter references | All sections | | 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | All sections | diff --git a/etc.txt b/etc.txt index 4de8893..eea4bfd 100644 --- a/etc.txt +++ b/etc.txt @@ -1,7 +1,7 @@ #!/usr/bin/env julia # Test script for mixed-content message testing # Tests receiving a mix of text, json, table, image, audio, video, and binary data -# from Julia serviceA to Julia serviceB using msghandler.jl smartreceive +# from Julia serviceA to Julia serviceB using msghandler.jl smartunpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. @@ -38,9 +38,9 @@ function test_mix_receive() log_trace("Received message on $(msg.subject)") incoming_msg = msg - # # Use msghandler.smartreceive to handle the data - # # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - # result = msghandler.smartreceive( + # # Use msghandler.smartunpack to handle the data + # # API: smartunpack(msg, download_handler; max_retries, base_delay, max_delay) + # result = msghandler.smartunpack( # msg; # max_retries = 5, # base_delay = 100, @@ -229,7 +229,7 @@ println("Note: This receiver will wait for messages from the sender.") println("Run test_julia_to_julia_mix_sender.jl first to send test data.") # Run receiver -println("\ntesting smartreceive for mixed content") +println("\ntesting smartunpack for mixed content") incoming_msg = test_mix_receive() println("\nTest completed.") @@ -250,7 +250,7 @@ println("\nTest completed.") Check architecture.md. For sending table I want to add JSON in addition to Apache Arrow. Currently I use "table" datatype when sending table data using Arrow. Now table that I want to send using JSON I will use "jsontable" as datatype while sending table using Arrow I will use "arrowtable" as datatype. -This will select how smartsend and smartreceive serialize/deserialize the table. +This will select how smartpack and smartunpack serialize/deserialize the table. Can you help me do this? Save the updated architecture.md into updated_architecture.md file. I will deal with source code later. diff --git a/examples/smartreceive_example.rs b/examples/smartreceive_example.rs index 56a36fd..7ae95b1 100644 --- a/examples/smartreceive_example.rs +++ b/examples/smartreceive_example.rs @@ -1,4 +1,4 @@ -use msghandler::{smartreceive, SmartreceiveOptions}; +use msghandler::{smartunpack, smartunpackOptions}; fn main() { // Simulated message JSON (received via any transport) @@ -40,9 +40,9 @@ fn main() { ] }"#; - let options = SmartreceiveOptions::default(); + let options = smartunpackOptions::default(); - match smartreceive(msg_json_str, &options) { + match smartunpack(msg_json_str, &options) { Ok(envelope) => { println!("=== Envelope Received ==="); println!("Correlation ID: {}", envelope.correlation_id); diff --git a/examples/smartsend_example.rs b/examples/smartsend_example.rs index e145b1f..8d244f1 100644 --- a/examples/smartsend_example.rs +++ b/examples/smartsend_example.rs @@ -1,4 +1,4 @@ -use msghandler::{smartsend, Payload, SmartsendOptions}; +use msghandler::{smartpack, Payload, smartpackOptions}; fn main() { // Create mixed payload data @@ -24,7 +24,7 @@ fn main() { ), ]; - let options = SmartsendOptions { + let options = smartpackOptions { broker_url: "localhost:4222".to_string(), fileserver_url: "http://localhost:8080".to_string(), msg_purpose: "chat".to_string(), @@ -32,7 +32,7 @@ fn main() { ..Default::default() }; - match smartsend("/agent/wine/api/v1/prompt", &payloads, &options) { + match smartpack("/agent/wine/api/v1/prompt", &payloads, &options) { Ok((envelope, json_str)) => { println!("=== Envelope Created ==="); println!("Correlation ID: {}", envelope.correlation_id); diff --git a/src/msghandler-csr.js b/src/msghandler-csr.js index 9c07a3c..5b075ce 100644 --- a/src/msghandler-csr.js +++ b/src/msghandler-csr.js @@ -378,13 +378,13 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * * @example * // Send a single payload - * const [env, envJsonStr] = await msghandlerCSR.smartsend( + * const [env, envJsonStr] = await msghandlerCSR.smartpack( * "/test", * [["dataname1", data1, "dictionary"]] * ); * * // Send multiple payloads (use jsontable instead of arrowtable for browser) - * const [env, envJsonStr] = await msghandlerCSR.smartsend( + * const [env, envJsonStr] = await msghandlerCSR.smartpack( * "/test", * [ * ["dataname1", data1, "dictionary"], @@ -395,7 +395,7 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * // Publish via your transport (NATS, MQTT, HTTP, etc.) * // await myNatsClient.publish("/test", envJsonStr); */ -async function smartsend(subject, data, options = {}) { +async function smartpack(subject, data, options = {}) { const { broker_url = DEFAULT_BROKER_URL, fileserver_url = DEFAULT_FILESERVER_URL, @@ -412,20 +412,20 @@ async function smartsend(subject, data, options = {}) { sender_id = uuidv4() } = options; - logTrace(correlation_id, `Starting smartsend for subject: ${subject}`); - logTrace(correlation_id, `smartsend: data array length=${data.length}`); + logTrace(correlation_id, `Starting smartpack for subject: ${subject}`); + logTrace(correlation_id, `smartpack: data array length=${data.length}`); // Debug: Log input data structure for (let i = 0; i < data.length; i++) { const [dataname, payloadData, payloadType] = data[i]; - logTrace(correlation_id, `smartsend: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); } // Process payloads const payloads = []; for (const [dataname, payloadData, payloadType] of data) { - logTrace(correlation_id, `smartsend: Processing payload '${dataname}' type=${payloadType}`); - logTrace(correlation_id, `smartsend: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: Processing payload '${dataname}' type=${payloadType}`); + logTrace(correlation_id, `smartpack: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); const payloadBytes = await serializeData(payloadData, payloadType); const payloadSize = payloadBytes.byteLength; @@ -502,7 +502,7 @@ async function smartsend(subject, data, options = {}) { * * @example * // Receive from JSON string directly - * const env = await msghandlerCSR.smartreceive(jsonString, { + * const env = await msghandlerCSR.smartunpack(jsonString, { * fileserver_download_handler: msghandlerCSR.fetchWithBackoff, * max_retries: 5, * base_delay: 100, @@ -510,7 +510,7 @@ async function smartsend(subject, data, options = {}) { * }); * * // Receive from transport message object (e.g., NATS, MQTT) - * const env = await msghandlerCSR.smartreceive(natsMsg, { + * const env = await msghandlerCSR.smartunpack(natsMsg, { * fileserver_download_handler: msghandlerCSR.fetchWithBackoff * }); * // env.payloads is an Array of [dataname, data, type] arrays @@ -518,7 +518,7 @@ async function smartsend(subject, data, options = {}) { * console.log(`${dataname}: ${data} (type: ${type})`); * } */ -async function smartreceive(msg, options = {}) { +async function smartunpack(msg, options = {}) { const { fileserver_download_handler = fetchWithBackoff, max_retries = 5, @@ -542,28 +542,28 @@ async function smartreceive(msg, options = {}) { throw new Error('Invalid message format: expected JSON string or message object'); } - logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); + logTrace('smartunpack', `smartunpack: raw payload length=${payload.length}`); // Debug: Show first 200 chars of payload const payloadPreview = payload.substring(0, 200); - logTrace('smartreceive', `smartreceive: payload preview: ${payloadPreview}`); + logTrace('smartunpack', `smartunpack: payload preview: ${payloadPreview}`); let envJsonObj; try { envJsonObj = JSON.parse(payload); } catch (e) { - logTrace('smartreceive', `smartreceive: JSON parse failed: ${e.message}`); + logTrace('smartunpack', `smartunpack: JSON parse failed: ${e.message}`); throw e; } logTrace(envJsonObj.correlation_id, 'Processing received message'); - logTrace(envJsonObj.correlation_id, `smartreceive: envelope has ${envJsonObj.payloads.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: envelope has ${envJsonObj.payloads.length} payloads`); // Process all payloads in the envelope const payloadsList = []; const numPayloads = envJsonObj.payloads.length; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing ${numPayloads} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing ${numPayloads} payloads`); for (let i = 0; i < numPayloads; i++) { const payloadObj = envJsonObj.payloads[i]; @@ -571,7 +571,7 @@ async function smartreceive(msg, options = {}) { const dataname = payloadObj.dataname; const payloadType = payloadObj.payload_type; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); if (transport === 'direct') { logTrace(envJsonObj.correlation_id, `Direct transport - decoding payload '${dataname}'`); @@ -614,7 +614,7 @@ async function smartreceive(msg, options = {}) { } } - logTrace(envJsonObj.correlation_id, `smartreceive: Successfully processed all ${payloadsList.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Successfully processed all ${payloadsList.length} payloads`); envJsonObj.payloads = payloadsList; return envJsonObj; } @@ -625,12 +625,12 @@ const msghandlerCSR = { /** * Send data with automatic transport selection */ - smartsend, + smartpack, /** * Receive and process messages */ - smartreceive, + smartunpack, /** * Upload data to plik server in one-shot mode diff --git a/src/msghandler.jl b/src/msghandler.jl index 859c4d1..87fcc2f 100644 --- a/src/msghandler.jl +++ b/src/msghandler.jl @@ -1,5 +1,5 @@ # Bi-Directional Data Bridge - Julia Module -# Implements smartsend and smartreceive for message transport +# Implements smartpack and smartunpack for message transport # This module provides functionality for sending and receiving data across network boundaries # with support for both direct payload transport and # URL-based transport for larger payloads. @@ -24,10 +24,10 @@ # # API Standard: # ```jldoctest -# # Input format for smartsend (always a list of tuples with type info) +# # Input format for smartpack (always a list of tuples with type info) # [(dataname1, data1, type1), (dataname2, data2, type2), ...] # -# # Output format for smartreceive (always returns a list of tuples) +# # Output format for smartunpack (always returns a list of tuples) # [(dataname1, data1, type1), (dataname2, data2, type2), ...] # ``` # @@ -337,7 +337,7 @@ function log_trace(correlation_id::String, message::String) end -""" smartsend - Send data with automatic transport selection, depending on payload size +""" smartpack - Send data with automatic transport selection, depending on payload size This function intelligently routes data delivery based on payload size relative to a threshold. If the serialized payload is smaller than `size_threshold`, it encodes the data as Base64 and constructs a "direct" msg_payload_v1. @@ -392,23 +392,23 @@ using UUIDs # Send a single payload (still wrapped in a list) data = Dict("key" => "value") -env, msg_json = smartsend("my.subject", [("dataname1", data, "dictionary")]) +env, msg_json = smartpack("my.subject", [("dataname1", data, "dictionary")]) # Send multiple payloads in one message with different types data1 = Dict("key1" => "value1") data2 = rand(10_000) # Small array -env, msg_json = smartsend("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")]) +env, msg_json = smartpack("my.subject", [("dataname1", data1, "dictionary"), ("dataname2", data2, "arrowtable")]) # Send a large array using fileserver upload data = rand(10_000_000) # ~80 MB -env, msg_json = smartsend("large.data", [("large_arrow_table", data, "arrowtable")]) +env, msg_json = smartpack("large.data", [("large_arrow_table", data, "arrowtable")]) # Send jsontable (JSON format) rows = [Dict("id" => 1, "name" => "Alice"), Dict("id" => 2, "name" => "Bob")] -env, msg_json = smartsend("json.data", [("users", rows, "jsontable")]) +env, msg_json = smartpack("json.data", [("users", rows, "jsontable")]) # Mixed content (e.g., chat with text and image) -env, msg_json = smartsend("chat.subject", [ +env, msg_json = smartpack("chat.subject", [ ("message_text", "Hello!", "text"), ("user_image", image_data, "image"), ("audio_clip", audio_data, "audio") @@ -419,8 +419,8 @@ env, msg_json = smartsend("chat.subject", [ # my_transport.publish(conn, subject, env_json_str) ``` """ -function smartsend( - subject::String, # smartreceive's subject +function smartpack( + subject::String, # smartunpack's subject data::AbstractArray{Tuple{String, T1, String}, 1}; # List of (dataname, data, type) tuples. Use Tuple{String, Any, String}[] for empty payloads broker_url::String = DEFAULT_BROKER_URL, # Broker URL fileserver_url = DEFAULT_FILESERVER_URL, @@ -446,7 +446,7 @@ function smartsend( )::Tuple{msg_envelope_v1, String} where {T1<:Any} # Log start of send operation - log_trace(correlation_id, "Starting smartsend for subject: $subject") + log_trace(correlation_id, "Starting smartpack for subject: $subject") # Process each payload in the list payloads = msg_payload_v1[] @@ -772,7 +772,7 @@ end # end -""" smartreceive - Receive and process messages +""" smartunpack - Receive and process messages This function processes incoming messages, handling both direct transport (base64 decoded payloads) and link transport (URL-based payloads). It deserializes the data based on the transport type and returns the result. @@ -801,11 +801,11 @@ A HTTP file server is required along with its download function. ```jldoctest # Receive and process message msg_json_str = String(msg.payload) -env = smartreceive(msg_json_str; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) +env = smartunpack(msg_json_str; fileserver_download_handler=_fetch_with_backoff, max_retries=5, base_delay=100, max_delay=5000) # env["payloads"] = [("dataname1", data1, "type1"), ("dataname2", data2, "type2"), ...] ``` """ -function smartreceive( +function smartunpack( msg_json_str::String; # get it from String(nats_msg.payload) fileserver_download_handler::Function = _fetch_with_backoff, max_retries::Int = 5, diff --git a/src/msghandler.js b/src/msghandler.js index d8b8fcc..ae97d8b 100644 --- a/src/msghandler.js +++ b/src/msghandler.js @@ -431,13 +431,13 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * * @example * // Send a single payload - * const [env, envJsonStr] = await smartsend( + * const [env, envJsonStr] = await smartpack( * "/test", * [["dataname1", data1, "dictionary"]] * ); * * // Send multiple payloads - * const [env, envJsonStr] = await smartsend( + * const [env, envJsonStr] = await smartpack( * "/test", * [ * ["dataname1", data1, "dictionary"], @@ -448,7 +448,7 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) { * // Publish via your transport (NATS, MQTT, HTTP, etc.) * // await myNatsClient.publish("/test", envJsonStr); */ -async function smartsend(subject, data, options = {}) { +async function smartpack(subject, data, options = {}) { const { broker_url = DEFAULT_BROKER_URL, fileserver_url = DEFAULT_FILESERVER_URL, @@ -465,20 +465,20 @@ async function smartsend(subject, data, options = {}) { sender_id = uuidv4() } = options; - logTrace(correlation_id, `Starting smartsend for subject: ${subject}`); - logTrace(correlation_id, `smartsend: data array length=${data.length}`); + logTrace(correlation_id, `Starting smartpack for subject: ${subject}`); + logTrace(correlation_id, `smartpack: data array length=${data.length}`); // Debug: Log input data structure for (let i = 0; i < data.length; i++) { const [dataname, payloadData, payloadType] = data[i]; - logTrace(correlation_id, `smartsend: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: payload[${i}] dataname=${dataname}, type=${payloadType}, data type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); } // Process payloads const payloads = []; for (const [dataname, payloadData, payloadType] of data) { - logTrace(correlation_id, `smartsend: Processing payload '${dataname}' type=${payloadType}`); - logTrace(correlation_id, `smartsend: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); + logTrace(correlation_id, `smartpack: Processing payload '${dataname}' type=${payloadType}`); + logTrace(correlation_id, `smartpack: payloadData type=${typeof payloadData}, constructor=${payloadData?.constructor?.name}`); const payloadBytes = await serializeData(payloadData, payloadType); const payloadSize = payloadBytes.byteLength; @@ -552,7 +552,7 @@ async function smartsend(subject, data, options = {}) { * * @example * // Receive from JSON string directly - * const env = await smartreceive(jsonString, { + * const env = await smartunpack(jsonString, { * fileserver_download_handler: fetchWithBackoff, * max_retries: 5, * base_delay: 100, @@ -560,7 +560,7 @@ async function smartsend(subject, data, options = {}) { * }); * * // Receive from transport message object (e.g., NATS, MQTT) - * const env = await smartreceive(natsMsg, { + * const env = await smartunpack(natsMsg, { * fileserver_download_handler: fetchWithBackoff * }); * // env.payloads is an Array of [dataname, data, type] arrays @@ -568,7 +568,7 @@ async function smartsend(subject, data, options = {}) { * console.log(`${dataname}: ${data} (type: ${type})`); * } */ -async function smartreceive(msg, options = {}) { +async function smartunpack(msg, options = {}) { const { fileserver_download_handler = fetchWithBackoff, max_retries = 5, @@ -592,28 +592,28 @@ async function smartreceive(msg, options = {}) { throw new Error('Invalid message format: expected JSON string or message object'); } - logTrace('smartreceive', `smartreceive: raw payload length=${payload.length}`); + logTrace('smartunpack', `smartunpack: raw payload length=${payload.length}`); // Debug: Show first 200 chars of payload const payloadPreview = payload.substring(0, 200); - logTrace('smartreceive', `smartreceive: payload preview: ${payloadPreview}`); + logTrace('smartunpack', `smartunpack: payload preview: ${payloadPreview}`); let envJsonObj; try { envJsonObj = JSON.parse(payload); } catch (e) { - logTrace('smartreceive', `smartreceive: JSON parse failed: ${e.message}`); + logTrace('smartunpack', `smartunpack: JSON parse failed: ${e.message}`); throw e; } logTrace(envJsonObj.correlation_id, 'Processing received message'); - logTrace(envJsonObj.correlation_id, `smartreceive: envelope has ${envJsonObj.payloads.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: envelope has ${envJsonObj.payloads.length} payloads`); // Process all payloads in the envelope const payloadsList = []; const numPayloads = envJsonObj.payloads.length; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing ${numPayloads} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing ${numPayloads} payloads`); for (let i = 0; i < numPayloads; i++) { const payloadObj = envJsonObj.payloads[i]; @@ -621,7 +621,7 @@ async function smartreceive(msg, options = {}) { const dataname = payloadObj.dataname; const payloadType = payloadObj.payload_type; - logTrace(envJsonObj.correlation_id, `smartreceive: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); + logTrace(envJsonObj.correlation_id, `smartunpack: Processing payload ${i + 1}/${numPayloads}: dataname=${dataname}, type=${payloadType}, transport=${transport}`); if (transport === 'direct') { logTrace(envJsonObj.correlation_id, `Direct transport - decoding payload '${dataname}'`); @@ -664,7 +664,7 @@ async function smartreceive(msg, options = {}) { } } - logTrace(envJsonObj.correlation_id, `smartreceive: Successfully processed all ${payloadsList.length} payloads`); + logTrace(envJsonObj.correlation_id, `smartunpack: Successfully processed all ${payloadsList.length} payloads`); envJsonObj.payloads = payloadsList; return envJsonObj; } @@ -675,12 +675,12 @@ const msghandler = { /** * Send data with automatic transport selection */ - smartsend, + smartpack, /** * Receive and process messages */ - smartreceive, + smartunpack, /** * Upload data to plik server in one-shot mode diff --git a/src/msghandler.py b/src/msghandler.py index f29f131..032ba36 100644 --- a/src/msghandler.py +++ b/src/msghandler.py @@ -372,7 +372,7 @@ def _build_payload( } -async def smartsend( +async def smartpack( subject: str, data: List[Tuple[str, Any, str]], broker_url: str = DEFAULT_BROKER_URL, @@ -429,7 +429,7 @@ async def smartsend( Example: >>> # Send a single payload (still wrapped in a list) >>> data = {"key": "value"} - >>> env, env_json_str = await smartsend( + >>> env, env_json_str = await smartpack( ... "my.subject", ... [("dataname1", data, "dictionary")] ... ) @@ -444,7 +444,7 @@ async def smartsend( if sender_id is None: sender_id = str(uuid.uuid4()) - log_trace(correlation_id, f"Starting smartsend for subject: {subject}") + log_trace(correlation_id, f"Starting smartpack for subject: {subject}") # Process payloads payloads = [] @@ -494,7 +494,7 @@ async def smartsend( return env, env_json_str -async def smartreceive( +async def smartunpack( msg: Any, fileserver_download_handler: Callable = fetch_with_backoff, max_retries: int = 5, @@ -521,10 +521,10 @@ async def smartreceive( Example: >>> # Receive from JSON string directly - >>> env = await smartreceive(json_string) + >>> env = await smartunpack(json_string) >>> >>> # Receive from transport message object (e.g., NATS, MQTT) - >>> env = await smartreceive(nats_msg, fileserver_download_handler=fetch_with_backoff) + >>> env = await smartunpack(nats_msg, fileserver_download_handler=fetch_with_backoff) >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] >>> for dataname, data, type_ in env["payloads"]: >>> print(f"{dataname}: {data} (type: {type_})") @@ -623,7 +623,7 @@ class msghandler: self.broker_url = broker_url or self.DEFAULT_BROKER_URL self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL - async def smartsend( + async def smartpack( self, subject: str, data: List[Tuple[str, Any, str]], @@ -635,16 +635,16 @@ class msghandler: Args: subject: Subject/topic to send to data: List of (dataname, data, type) tuples - **kwargs: Additional options passed to smartsend + **kwargs: Additional options passed to smartpack Returns: Tuple of (env, env_json_str) """ kwargs['broker_url'] = kwargs.get('broker_url', self.broker_url) kwargs['fileserver_url'] = kwargs.get('fileserver_url', self.fileserver_url) - return await smartsend(subject, data, **kwargs) + return await smartpack(subject, data, **kwargs) - async def smartreceive( + async def smartunpack( self, msg: Any, **kwargs @@ -654,12 +654,12 @@ class msghandler: Args: msg: Message to process - **kwargs: Additional options passed to smartreceive + **kwargs: Additional options passed to smartunpack Returns: Dict with envelope metadata and payloads """ - return await smartreceive(msg, **kwargs) + return await smartunpack(msg, **kwargs) # Convenience functions for module-level usage @@ -679,7 +679,7 @@ def send( Returns: Tuple of (env, env_json_str) """ - return asyncio.run(smartsend(subject, data, **kwargs)) + return asyncio.run(smartpack(subject, data, **kwargs)) def receive( @@ -696,12 +696,12 @@ def receive( Returns: Dict with envelope metadata and payloads """ - return asyncio.run(smartreceive(msg, **kwargs)) + return asyncio.run(smartunpack(msg, **kwargs)) __all__ = [ - 'smartsend', - 'smartreceive', + 'smartpack', + 'smartunpack', 'plik_oneshot_upload', 'fetch_with_backoff', 'msghandler', diff --git a/src/msghandler.rs b/src/msghandler.rs index 102ca27..01801e8 100644 --- a/src/msghandler.rs +++ b/src/msghandler.rs @@ -1,6 +1,6 @@ // msghandler Rust Module // Cross-platform bi-directional data bridge -// Implements smartsend and smartreceive for message transport +// Implements smartpack and smartunpack for message transport // with support for both direct payload transport and URL-based transport // for larger payloads using the Claim-Check pattern. // @@ -325,8 +325,8 @@ impl MsgEnvelopeV1 { // Options Structures // ============================================================================ -/// Options for the `smartsend` function -pub struct SmartsendOptions { +/// Options for the `smartpack` function +pub struct smartpackOptions { /// Broker URL pub broker_url: String, /// HTTP file server URL for large payloads @@ -355,9 +355,9 @@ pub struct SmartsendOptions { pub sender_id: String, } -impl Default for SmartsendOptions { +impl Default for smartpackOptions { fn default() -> Self { - SmartsendOptions { + smartpackOptions { broker_url: DEFAULT_BROKER_URL.to_string(), fileserver_url: DEFAULT_FILESERVER_URL.to_string(), fileserver_upload_handler: None, @@ -375,8 +375,8 @@ impl Default for SmartsendOptions { } } -/// Options for the `smartreceive` function -pub struct SmartreceiveOptions { +/// Options for the `smartunpack` function +pub struct smartunpackOptions { /// Custom file server download handler (optional, uses exponential backoff by default) pub fileserver_download_handler: Option>, /// Maximum retry attempts for fetching a URL @@ -387,9 +387,9 @@ pub struct SmartreceiveOptions { pub max_delay: u64, } -impl Default for SmartreceiveOptions { +impl Default for smartunpackOptions { fn default() -> Self { - SmartreceiveOptions { + smartunpackOptions { fileserver_download_handler: None, max_retries: DEFAULT_MAX_RETRIES, base_delay: DEFAULT_BASE_DELAY, @@ -689,7 +689,7 @@ pub fn log_trace(correlation_id: &str, message: &str) { } // ============================================================================ -// Public API: smartsend +// Public API: smartpack // ============================================================================ /// Send data with automatic transport selection. @@ -715,23 +715,23 @@ pub fn log_trace(correlation_id: &str, message: &str) { /// /// # Example /// ```no_run -/// use msghandler::{smartsend, Payload, SmartsendOptions}; +/// use msghandler::{smartpack, Payload, smartpackOptions}; /// -/// let (envelope, json_str) = smartsend( +/// let (envelope, json_str) = smartpack( /// "/agent/wine/api/v1/prompt", /// &[ /// ("msg".to_string(), Payload::Text("Hello!".to_string()), "text".to_string()), /// ("data".to_string(), Payload::Binary(vec![1, 2, 3]), "binary".to_string()), /// ], -/// &SmartsendOptions::default(), +/// &smartpackOptions::default(), /// ).unwrap(); /// /// // Caller publishes via their preferred transport /// ``` -pub fn smartsend( +pub fn smartpack( subject: &str, data: &[(String, Payload, String)], - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { let correlation_id = if options.correlation_id.is_empty() { Uuid::new_v4().to_string() @@ -752,7 +752,7 @@ pub fn smartsend( }; log_trace(&correlation_id, &format!( - "Starting smartsend for subject: {}", subject + "Starting smartpack for subject: {}", subject )); let mut payloads: Vec = Vec::new(); @@ -844,7 +844,7 @@ pub fn smartsend( // ============================================================================ /// Store deserialized Payload data back into a MsgPayloadV1's data field. -/// After smartreceive(), payload.data contains the deserialized content as a string +/// After smartunpack(), payload.data contains the deserialized content as a string /// (decoded text, JSON string, or base64 for binary types). fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> MsgPayloadV1 { let mut p = payload.clone(); @@ -861,7 +861,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms } // ============================================================================ -// Public API: smartreceive +// Public API: smartunpack // ============================================================================ /// Receive and process messages. @@ -880,7 +880,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// /// # Example /// ```no_run -/// use msghandler::{smartreceive, SmartreceiveOptions}; +/// use msghandler::{smartunpack, smartunpackOptions}; /// use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; /// /// let msg_json_str = r#"{"correlation_id":"abc123","msg_id":"msg-uuid", @@ -893,7 +893,7 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// "data":"SGVsbG8=","metadata":{"payload_bytes":5} /// }]}"#; /// -/// let envelope = smartreceive(msg_json_str, &SmartreceiveOptions::default()).unwrap(); +/// let envelope = smartunpack(msg_json_str, &smartunpackOptions::default()).unwrap(); /// /// for payload in &envelope.payloads { /// if payload.transport == "direct" { @@ -904,9 +904,9 @@ fn store_deserialized_data(payload: &MsgPayloadV1, deserialized: &Payload) -> Ms /// } /// } /// ``` -pub fn smartreceive( +pub fn smartunpack( msg_json_str: &str, - options: &SmartreceiveOptions, + options: &smartunpackOptions, ) -> Result { // Parse the JSON envelope let mut env: MsgEnvelopeV1 = serde_json::from_str(msg_json_str) @@ -998,9 +998,9 @@ pub fn smartreceive( pub fn send_text( subject: &str, text: &str, - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { - smartsend( + smartpack( subject, &[( "text".to_string(), @@ -1015,9 +1015,9 @@ pub fn send_text( pub fn send_dictionary( subject: &str, data: &JsonValue, - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { - smartsend( + smartpack( subject, &[( "dictionary".to_string(), @@ -1032,9 +1032,9 @@ pub fn send_dictionary( pub fn send_binary( subject: &str, data: &[u8], - options: &SmartsendOptions, + options: &smartpackOptions, ) -> Result<(MsgEnvelopeV1, String), MsgHandlerError> { - smartsend( + smartpack( subject, &[( "binary".to_string(), @@ -1094,10 +1094,10 @@ pub fn plik_upload_file( // All public types are already exported via `pub` on their definitions. // Key types: -// - `smartsend`, `smartreceive` - main API functions +// - `smartpack`, `smartunpack` - main API functions // - `Payload` - type-safe payload enum // - `MsgEnvelopeV1`, `MsgPayloadV1` - wire format structs -// - `SmartsendOptions`, `SmartreceiveOptions` - configuration +// - `smartpackOptions`, `smartunpackOptions` - configuration // - `FileUploadHandler`, `FileDownloadHandler` - trait abstractions // - `PlikOneshotUploadHandler`, `BackoffDownloadHandler` - default implementations // - `MsgHandlerError` - error type @@ -1193,12 +1193,12 @@ mod tests { #[test] fn test_default_options() { - let opts = SmartsendOptions::default(); + let opts = smartpackOptions::default(); assert_eq!(opts.size_threshold, DEFAULT_SIZE_THRESHOLD); assert_eq!(opts.broker_url, DEFAULT_BROKER_URL); assert_eq!(opts.fileserver_url, DEFAULT_FILESERVER_URL); - let opts = SmartreceiveOptions::default(); + let opts = smartunpackOptions::default(); assert_eq!(opts.max_retries, DEFAULT_MAX_RETRIES); assert_eq!(opts.base_delay, DEFAULT_BASE_DELAY); assert_eq!(opts.max_delay, DEFAULT_MAX_DELAY); diff --git a/src/msghandler_mpy.py b/src/msghandler_mpy.py index e6ec6ab..99fb5db 100644 --- a/src/msghandler_mpy.py +++ b/src/msghandler_mpy.py @@ -263,7 +263,7 @@ def _publish(subject, message, correlation_id): # Placeholder - actual implementation would publish via preferred transport -def smartsend(subject, data, **kwargs): +def smartpack(subject, data, **kwargs): """ Send data with automatic transport selection. @@ -306,7 +306,7 @@ def smartsend(subject, data, **kwargs): Example: >>> # Send text payload - >>> env, env_json_str = smartsend( + >>> env, env_json_str = smartpack( ... "/chat", ... [("message", "Hello!", "text")] ... ) @@ -330,7 +330,7 @@ def smartsend(subject, data, **kwargs): is_publish = kwargs.get('is_publish', True) fileserver_upload_handler = kwargs.get('fileserver_upload_handler', _sync_fileserver_upload) - log_trace(correlation_id, f"Starting smartsend for subject: {subject}") + log_trace(correlation_id, f"Starting smartpack for subject: {subject}") # Process payloads payloads = [] @@ -390,7 +390,7 @@ def smartsend(subject, data, **kwargs): return env, env_json_str -def smartreceive(msg, **kwargs): +def smartunpack(msg, **kwargs): """ Receive and process messages. @@ -414,10 +414,10 @@ def smartreceive(msg, **kwargs): Example: >>> # Receive from JSON string - >>> env = smartreceive(json_string) + >>> env = smartunpack(json_string) >>> >>> # Receive from transport message object - >>> env = smartreceive(transport_msg, fileserver_download_handler=_sync_fileserver_download) + >>> env = smartunpack(transport_msg, fileserver_download_handler=_sync_fileserver_download) >>> # env is a Dict with "payloads" key containing List[Tuple[str, Any, str]] >>> for dataname, data, type_ in env["payloads"]: ... print(f"{dataname}: {data} (type: {type_})") @@ -530,34 +530,34 @@ class msghandler: self.broker_url = broker_url or self.DEFAULT_BROKER_URL self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL - def smartsend(self, subject, data, **kwargs): + def smartpack(self, subject, data, **kwargs): """ Send data. Args: subject: Subject/topic to send to data: List of (dataname, data, type) tuples - **kwargs: Additional options passed to smartsend + **kwargs: Additional options passed to smartpack Returns: Tuple of (env, env_json_str) """ kwargs['broker_url'] = kwargs.get('broker_url', self.broker_url) kwargs['fileserver_url'] = kwargs.get('fileserver_url', self.fileserver_url) - return smartsend(subject, data, **kwargs) + return smartpack(subject, data, **kwargs) - def smartreceive(self, msg, **kwargs): + def smartunpack(self, msg, **kwargs): """ Receive and process message. Args: msg: Message to process - **kwargs: Additional options passed to smartreceive + **kwargs: Additional options passed to smartunpack Returns: Dict with envelope metadata and payloads """ - return smartreceive(msg, **kwargs) + return smartunpack(msg, **kwargs) # Convenience functions for module-level usage @@ -573,7 +573,7 @@ def send(subject, data, **kwargs): Returns: Tuple of (env, env_json_str) """ - return smartsend(subject, data, **kwargs) + return smartpack(subject, data, **kwargs) def receive(msg, **kwargs): @@ -587,12 +587,12 @@ def receive(msg, **kwargs): Returns: Dict with envelope metadata and payloads """ - return smartreceive(msg, **kwargs) + return smartunpack(msg, **kwargs) __all__ = [ - 'smartsend', - 'smartreceive', + 'smartpack', + 'smartunpack', 'msghandler', 'send', 'receive', diff --git a/test/test_js_mix_payloads_receiver.js b/test/test_js_mix_payloads_receiver.js index 7c5b75b..6f79b39 100644 --- a/test/test_js_mix_payloads_receiver.js +++ b/test/test_js_mix_payloads_receiver.js @@ -1,6 +1,6 @@ /** * JavaScript Mix Payloads Receiver Test - * Tests the smartreceive function with mixed payload types + * Tests the smartunpack function with mixed payload types * * This test mirrors test_julia_mix_payloads_receiver.jl and demonstrates that * any combination and any number of mixed content can be received correctly. @@ -50,8 +50,8 @@ async function runTest() { console.log(`Received message on ${msg.subject}`); try { - // Process the message using smartreceive - const envelope = await msghandler.smartreceive(msg, { + // Process the message using smartunpack + const envelope = await msghandler.smartunpack(msg, { fileserver_download_handler: msghandler.fetchWithBackoff, max_retries: 5, base_delay: 100, diff --git a/test/test_js_mix_payloads_sender.js b/test/test_js_mix_payloads_sender.js index 7969732..c88eecb 100644 --- a/test/test_js_mix_payloads_sender.js +++ b/test/test_js_mix_payloads_sender.js @@ -1,6 +1,6 @@ /** * JavaScript Mix Payloads Sender Test - * Tests the smartsend function with mixed payload types + * Tests the smartpack function with mixed payload types * * This test mirrors test_julia_mix_payloads_sender.jl and demonstrates that * any combination and any number of mixed content can be sent correctly. @@ -169,7 +169,7 @@ async function runTest() { try { // Send the message console.log('Sending mixed payloads...\n'); - const [env, envJsonStr] = await msghandler.smartsend( + const [env, envJsonStr] = await msghandler.smartpack( TEST_SUBJECT, payloads, { diff --git a/test/test_julia_mix_payloads_receiver.jl b/test/test_julia_mix_payloads_receiver.jl index 673afe8..a88a67b 100644 --- a/test/test_julia_mix_payloads_receiver.jl +++ b/test/test_julia_mix_payloads_receiver.jl @@ -1,7 +1,7 @@ #!/usr/bin/env julia # Test script for mixed-content message testing # Tests receiving a mix of text, json, table, image, audio, video, and binary data -# from Julia serviceA to Julia serviceB using msghandler.jl smartreceive +# from Julia serviceA to Julia serviceB using msghandler.jl smartunpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. @@ -36,9 +36,9 @@ function test_mix_receive() NATS.subscribe(conn, SUBJECT) do msg log_trace("Received message on $(msg.subject)") - # Use msghandler.smartreceive to handle the data - # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) - result = msghandler.smartreceive( + # Use msghandler.smartunpack to handle the data + # API: smartunpack(msg, download_handler; max_retries, base_delay, max_delay) + result = msghandler.smartunpack( msg; max_retries = 5, base_delay = 100, @@ -245,7 +245,7 @@ println("Note: This receiver will wait for messages from the sender.") println("Run test_julia_to_julia_mix_sender.jl first to send test data.") # Run receiver -println("\ntesting smartreceive for mixed content") +println("\ntesting smartunpack for mixed content") test_mix_receive() println("\nTest completed.") \ No newline at end of file diff --git a/test/test_julia_mix_payloads_sender.jl b/test/test_julia_mix_payloads_sender.jl index d75b521..a7e77de 100644 --- a/test/test_julia_mix_payloads_sender.jl +++ b/test/test_julia_mix_payloads_sender.jl @@ -1,7 +1,7 @@ #!/usr/bin/env julia # Test script for mixed-content message testing # Tests sending a mix of text, dictionary, arrowtable, jsontable, image, audio, video, and binary data -# from Julia serviceA to Julia serviceB using msghandler.jl smartsend +# from Julia serviceA to Julia serviceB using msghandler.jl smartpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. @@ -166,7 +166,7 @@ function create_sample_data() end -# Sender: Send mixed content via smartsend +# Sender: Send mixed content via smartpack function test_mix_send() # Create sample data (text_data, dict_data, arrow_table_small, arrow_table_large, json_table_small, json_table_large, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data) = create_sample_data() @@ -203,8 +203,8 @@ function test_mix_send() ("binary_file_large", large_binary_data, "binary") ] - # Use smartsend with mixed content - sendinfo = msghandler.smartsend( + # Use smartpack with mixed content + sendinfo = msghandler.smartpack( SUBJECT, payloads; # List of (dataname, data, type) tuples broker_url = NATS_URL, @@ -251,7 +251,7 @@ println("Starting mixed-content transport test...") println("Correlation ID: $correlation_id") # Run sender -println("start smartsend for mixed content") +println("start smartpack for mixed content") test_mix_send() println("\nTest completed.") diff --git a/test/test_py_mix_payloads_sender.py b/test/test_py_mix_payloads_sender.py index 20a9d7f..ea5e798 100644 --- a/test/test_py_mix_payloads_sender.py +++ b/test/test_py_mix_payloads_sender.py @@ -1,6 +1,6 @@ """ Python Mix Payloads Sender Test -Tests the smartsend function with mixed payload types +Tests the smartpack function with mixed payload types """ import asyncio @@ -11,7 +11,7 @@ import base64 # Add parent directory to path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from msghandler import smartsend, DEFAULT_BROKER_URL, DEFAULT_FILESERVER_URL +from msghandler import smartpack, DEFAULT_BROKER_URL, DEFAULT_FILESERVER_URL TEST_SUBJECT = '/test/mix' TEST_BROKER_URL = os.environ.get('NATS_URL', 'nats://localhost:4222') @@ -56,7 +56,7 @@ async def run_test(): try: # Send the message print('Sending mixed payloads...') - env, env_json_str = await smartsend( + env, env_json_str = await smartpack( TEST_SUBJECT, test_data, broker_url=TEST_BROKER_URL, @@ -164,7 +164,7 @@ async def run_test(): ('audio', bytes([0x46, 0x4C, 0x41, 0x43]), 'audio') ] - chat_env, _ = await smartsend( + chat_env, _ = await smartpack( TEST_SUBJECT, chat_data, broker_url=TEST_BROKER_URL, From 312d14b28ff369c744cc362bb8a57914c9564a12 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 06:17:30 +0700 Subject: [PATCH 05/14] update docs --- docs/architecture.md | 942 ------------------------------------ docs/implementation-plan.md | 416 ++++++++++++++++ docs/requirements.md | 143 ++++-- docs/solution-design.md | 345 +++++++++++++ 4 files changed, 851 insertions(+), 995 deletions(-) delete mode 100644 docs/architecture.md create mode 100644 docs/implementation-plan.md create mode 100644 docs/solution-design.md diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 1259106..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,942 +0,0 @@ -# Architecture Documentation: msghandler - -**Version**: 1.4.0 -**Date**: 2026-05-14 -**Status**: Active -**Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) -**Architecture Level**: C4 Container Level - ---- - -## 1. Executive Summary - -This document defines the **blueprint** for msghandler - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using a message broker as the transport layer. - -This architecture document serves as the single source of truth for: -- **System Structure**: How components fit together and interact -- **Scaling Considerations**: How the system scales horizontally and vertically -- **Failure Modes**: How the system handles failures and recovers -- **Trade-off Decisions**: The rationale behind architectural decisions - -### 1.1 Specification Traceability - -| Architecture Section | Specification Reference | UI Specification Reference | Requirement ID(s) | -|---------------------|-------------------------|---------------------------|-------------------| -| Section 2 (Context Diagram) | specification.md:2 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | -| Section 3 (Container Diagram) | specification.md:2, specification.md:3, specification.md:11 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | -| Section 4 (Component Diagram) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | -| Section 5 (High-Level) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | -| Section 6 (Message Envelope) | specification.md:2, specification.md:3, specification.md:8 | - | FR-011, FR-012, FR-013, FR-014, NFR-401, NFR-403 | -| Section 7 (Payload Type) | specification.md:3, specification.md:5, specification.md:6 | - | FR-001, FR-002, FR-003, FR-006, FR-012, NFR-101, NFR-102 | -| Section 8 (Transport Strategy) | specification.md:6, specification.md:7 | - | FR-003, FR-004, FR-005, FR-010, NFR-104, NFR-105, NFR-106 | -| Section 9 (Platform-Specific) | specification.md:13, specification.md:14 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | -| Section 10 (Scaling) | specification.md:7, specification.md:13 | - | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 | -| Section 11 (Failure Modes) | specification.md:9, specification.md:11 | - | FR-008, FR-009, FR-010, FR-011, NFR-201, NFR-202, NFR-203 | -| Section 12 (Trade-offs) | specification.md:2, specification.md:3, specification.md:6, specification.md:7 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | -| Section 13 (Deployment) | specification.md:12, specification.md:18 | - | FR-013, FR-014, NFR-201, NFR-203 | -| Section 14 (Security) | specification.md:4, specification.md:9, specification.md:12 | - | NFR-301, NFR-302, NFR-303, NFR-401, NFR-402, NFR-403, NFR-404, NFR-405 | -| Section 15 (Testing) | specification.md:17 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | - ---- - -## 2. Architecture Overview - -## Architecture Overview - -### C4 Context Diagram - -```mermaid -flowchart TD - subgraph "External Systems" - Message_Broker[Message Broker
NATS/MQTT/WebSocket/Custom] - File_Server[HTTP File Server
Plik/AWS S3/Custom] - end - - Julia_App[Julia Application] - JS_App[JavaScript Application
Node.js/Browser] - Python_App[Python Application
Desktop] - Dart_App[Dart Application
Desktop/Flutter/Web] - Rust_App[Rust Application
Server/Desktop] - MicroPython_App[MicroPython Device] - end - - Julia_App -->|Transport| Message_Broker - JS_App -->|Transport| Message_Broker - Python_App -->|Transport| Message_Broker - Dart_App -->|Transport| Message_Broker - Rust_App -->|Transport| Message_Broker - MicroPython_App -->|Transport| Message_Broker - - Julia_App -->|HTTP| File_Server - JS_App -->|HTTP| File_Server - Python_App -->|HTTP| File_Server - Dart_App -->|HTTP| File_Server - Rust_App -->|HTTP| File_Server - MicroPython_App -->|HTTP| File_Server - - style Message_Broker fill:#fff3e0,stroke:#f57c00 - style File_Server fill:#f3e5f5,stroke:#9c27b4 - style Julia_App fill:#e8f5e9,stroke:#4caf50 - style JS_App fill:#e3f2fd,stroke:#2196f3 - style Python_App fill:#e3f2fd,stroke:#2196f3 - style Dart_App fill:#fff0f6,stroke:#e91e63 - style Rust_App fill:#dea584,stroke:#e65100 - style MicroPython_App fill:#fce4ec,stroke:#e91e63 -``` - -### C4 Container Diagram - -```mermaid -flowchart TD - subgraph "Client Container" - Julia_Module[Julia msghandler Module] - JS_Module[JavaScript msghandler Module] - Python_Module[Python msghandler Module] - Dart_Module[Dart msghandler Module] - Rust_Module[Rust msghandler Module] - MicroPython_Module[MicroPython msghandler Module] - end - - Julia_Module --> Transport_Client - JS_Module --> Transport_Client - Python_Module --> Transport_Client - Dart_Module --> Transport_Client - Rust_Module --> Transport_Client - MicroPython_Module --> Transport_Client - - Transport_Client --> Message_Broker - - Julia_Module --> File_Client - JS_Module --> File_Client - Python_Module --> File_Client - Dart_Module --> File_Client - Rust_Module --> File_Client - MicroPython_Module --> File_Client - - File_Client --> File_Server - - style Julia_Module fill:#e8f5e9,stroke:#4caf50 - style JS_Module fill:#e3f2fd,stroke:#2196f3 - style Python_Module fill:#e3f2fd,stroke:#2196f3 - style Dart_Module fill:#fff0f6,stroke:#e91e63 - style Rust_Module fill:#dea584,stroke:#e65100 - style MicroPython_Module fill:#fce4ec,stroke:#e91e63 - style Message_Broker fill:#fff3e0,stroke:#f57c00 - style File_Server fill:#f3e5f5,stroke:#9c27b4 -``` - -### C4 Component Diagram (Julia Implementation) - -```mermaid -flowchart TD - subgraph "msghandler Module" - smartpack[smartpack Function] - smartunpack[smartunpack Function] - - Serialize[_serialize_data] - Deserialize[_deserialize_data] - - EnvelopeToJson[envelope_to_json] - - FileServerUpload[fileserver_upload_handler] - FileServerDownload[fileserver_download_handler] - - LogTrace[log_trace] - end - - subgraph "Data Models" - Payload[msg_payload_v1 Struct] - Envelope[msg_envelope_v1 Struct] - end - - smartpack --> Serialize - smartpack --> EnvelopeToJson - smartpack --> FileServerUpload - - smartunpack --> Deserialize - smartunpack --> FileServerDownload - - EnvelopeToJson --> Envelope - Serialize --> Payload - - style smartpack fill:#d1fae5,stroke:#10b981 - style smartunpack fill:#d1fae5,stroke:#10b981 - style FileServerUpload fill:#fef3c7,stroke:#f59e0b - style FileServerDownload fill:#fef3c7,stroke:#f59e0b -``` - ---- - -## High-Level Architecture - -### System Components - -| Component | Purpose | Platform Support | -|-----------|---------|------------------| -| **smartpack** | Send data with automatic transport selection, returns (envelope, json_string) for caller to publish via transport | All | -| **smartunpack** | Receive and process messages from JSON string | All | -| **_serialize_data** | Serialize data according to payload type | All | -| **_deserialize_data** | Deserialize bytes to native data types | All | -| **envelope_to_json** | Convert msg_envelope_v1 struct to JSON string | All | -| **log_trace** | Log trace messages with correlation ID | All | -| **fileserver_upload_handler** | Upload large payloads to HTTP server | Desktop (Julia/JS/Python/Dart/Rust) | -| **fileserver_download_handler** | Download payloads from HTTP server with exponential backoff | Desktop (Julia/JS/Python/Dart/Rust) | -| **plik_upload_file** | Upload a local file to Plik server from disk | Rust | - -### Data Flow - -```mermaid -flowchart TD - A[User calls smartpack subject data] --> B[Process each payload] - B --> C{Calculate serialized size} - C -->|Size < Threshold| D[Direct Transport] - C -->|Size >= Threshold| E[Link Transport] - - D --> F[Serialize data] - F --> G[Base64 encode] - G --> H[Build payload object] - - E --> I[Serialize data] - I --> J[Upload to file server] - J --> K[Get download URL] - K --> H - - H --> L[Build envelope] - L --> M[Convert to JSON] - M --> N[Return envelope + JSON to caller] - - style A fill:#f9f9f9,stroke:#333 - style N fill:#e0e7ff,stroke:#3b82f6 - style D fill:#d1fae5,stroke:#10b981 - style E fill:#fef3c7,stroke:#f59e0b -``` - ---- - -## Message Envelope Architecture - -### msg_envelope_v1 Structure (Julia) - -```julia -struct msg_envelope_v1 - correlation_id::String # UUID v4 for distributed tracing - msg_id::String # UUID v4 for this message - timestamp::String # ISO 8601 UTC timestamp - - send_to::String # Topic/subject to publish to - msg_purpose::String # ACK, NACK, updateStatus, shutdown, chat - sender_name::String # Sender application name - sender_id::String # UUID v4 of sender - receiver_name::String # Receiver application name (empty = broadcast) - receiver_id::String # UUID v4 of receiver (empty = broadcast) - - reply_to::String # Topic for reply messages - reply_to_msg_id::String # Message ID being replied to - broker_url::String # Broker URL for the transport layer - - metadata::Dict{String, Any} # Message-level metadata - payloads::Vector{msg_payload_v1} # List of payloads -end -``` - -### msg_payload_v1 Structure (Julia) - -```julia -struct msg_payload_v1 - id::String # UUID v4 for this payload - dataname::String # Name of the payload - payload_type::String # text, dictionary, arrowtable, etc. - transport::String # direct or link - encoding::String # none, json, base64, arrow-ipc - size::Integer # Size in bytes - data::Any # Base64 string or URL - metadata::Dict{String, Any} # Payload-level metadata -end -``` - -### JSON Schema (Cross-Platform) - -```json -{ - "correlation_id": "string (UUID v4)", - "msg_id": "string (UUID v4)", - "timestamp": "string (ISO 8601 UTC)", - "send_to": "string", - "msg_purpose": "string", - "sender_name": "string", - "sender_id": "string (UUID v4)", - "receiver_name": "string", - "receiver_id": "string (UUID v4)", - "reply_to": "string", - "reply_to_msg_id": "string", - "broker_url": "string", - "metadata": "object", - "payloads": [ - { - "id": "string (UUID v4)", - "dataname": "string", - "payload_type": "string", - "transport": "string", - "encoding": "string", - "size": "integer", - "data": "string or URL", - "metadata": "object" - } - ] -} -``` - ---- - -## Payload Type Architecture - -### Supported Payload Types - -| Type | Description | Serialization | Encoding | Platforms | -|------|-------------|---------------|----------|-----------| -| `text` | Plain text string | UTF-8 bytes | Base64 | All | -| `dictionary` | JSON object | JSON string | Base64/JSON | All | -| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop (Julia/Python/Node.js/Dart/Rust) | -| `jsontable` | JSON array of objects | JSON string | Base64/json | All (including Browser/Dart Web) | -| `image` | Binary image data | Raw bytes | Base64 | All | -| `audio` | Binary audio data | Raw bytes | Base64 | All | -| `video` | Binary video data | Raw bytes | Base64 | All | -| `binary` | Generic binary data | Raw bytes | Base64 | All | - -### Serialization Logic - -```mermaid -flowchart TD - A[Input data + payload_type] --> B{Payload Type} - - B -->|"text"| C[UTF-8 encode] - B -->|"dictionary"| D[JSON serialize] - B -->|"arrowtable"| E[Arrow IPC serialize] - B -->|"jsontable"| F[JSON serialize] - B -->|"image"| G[Raw bytes] - B -->|"audio"| H[Raw bytes] - B -->|"video"| I[Raw bytes] - B -->|"binary"| J[Raw bytes] - - C --> K[Return bytes] - D --> K - E --> K - F --> K - G --> K - H --> K - I --> K - J --> K - - style A fill:#f9f9f9,stroke:#333 - style K fill:#e0e7ff,stroke:#3b82f6 -``` - ---- - -## Transport Strategy Architecture - -### Size Threshold Decision Logic - -| Platform | Size Threshold | Notes | -|----------|----------------|-------| -| Desktop (Julia/JS/Python/Dart) | 500,000 bytes (0.5MB) | Default threshold | -| Dart Desktop | 500,000 bytes (0.5MB) | Default threshold | -| Dart Flutter | 500,000 bytes (0.5MB) | Default threshold | -| Dart Web | 500,000 bytes (0.5MB) | Default threshold | -| MicroPython | 100,000 bytes (100KB) | Lower threshold for memory constraints | - -### Transport Selection Flow - -```mermaid -flowchart TD - A[smartpack called] --> B[Serialize payload] - B --> C[Calculate size] - C --> D{Size < Threshold?} - - D -->|Yes| E[Direct Transport] - D -->|No| F[Link Transport] - - E --> G[Base64 encode] - G --> H[Build payload with direct transport] - - F --> I[Upload to file server] - I --> J[Get download URL] - J --> K[Build payload with link transport] - - H --> L[Build envelope] - K --> L - - style A fill:#f9f9f9,stroke:#333 - style L fill:#e0e7ff,stroke:#3b82f6 - style E fill:#d1fae5,stroke:#10b981 - style F fill:#fef3c7,stroke:#f59e0b -``` - -### Direct Transport Protocol - -When `transport = "direct"`, the `data` field contains a Base64-encoded string of the serialized payload. - -**Encoding Rules**: -- `text`: UTF-8 → Base64 -- `dictionary`: JSON → Base64 (or direct JSON) -- `arrowtable`: Arrow IPC → Base64 (or arrow-ipc) -- `jsontable`: JSON → Base64 (or direct JSON) -- `image`/`audio`/`video`/`binary`: Raw bytes → Base64 - -### Link Transport Protocol - -When `transport = "link"`, the `data` field contains a URL pointing to the uploaded payload. - -**Upload Flow**: -1. Serialize payload according to `payload_type` -2. Upload to HTTP file server (e.g., Plik) -3. Include returned URL in `data` field - -**Download Flow**: -1. Extract URL from payload -2. Fetch with exponential backoff (max 5 retries) -3. Deserialize based on `payload_type` - ---- - -## Platform-Specific Architecture - -### Julia Architecture - -Julia leverages multiple dispatch for type-specific implementations: - -- **Multiple Dispatch**: Function overloading based on argument types -- **Struct-based Data Models**: Explicit type definitions with `struct` -- **Native Arrow IPC**: Support via `Arrow.jl` -- **Async/Await**: Tasks for non-blocking I/O - -```julia -# Multiple dispatch for serialization -function _serialize_data(data::String, payload_type::String) - # Text serialization -end - -function _serialize_data(data::Dict, payload_type::String) - # Dictionary serialization -end - -function _serialize_data(data::DataFrame, payload_type::String) - # Arrow table serialization -end -``` - -### JavaScript Architecture - -JavaScript uses async/await for non-blocking I/O: - -- **Module-level Utilities**: Serialization functions -- **Native ArrayBuffer**: Binary data handling (Browser) / Buffer (Node.js) -- **Fetch API**: HTTP file server communication - -#### Node.js Implementation (msghandler_ssr.js) - -- **Transport connections**: Uses broker URLs (e.g., `nats://`, `mqtt://`, `ws://`) -- **Apache Arrow IPC**: Full support via `apache-arrow` -- **Buffer for binary data**: Native Node.js Buffer handling - -#### Browser Implementation (msghandler_csr.js) - -- **WebSocket connections**: Uses `ws://` or `wss://` URLs (transport-agnostic) -- **No Apache Arrow**: Uses `jsontable` for tabular data only -- **Uint8Array for binary data**: Browser-compatible binary handling -- **Web Crypto API**: UUID generation via `crypto.getRandomValues()` - -### Python Architecture - -Python uses classes for stateful operations: - -- **Class-based msghandler**: Encapsulated API -- **Dataclasses**: Structured data (MsgPayloadV1, MsgEnvelopeV1) -- **Async/await**: I/O operations -- **pyarrow**: Arrow IPC support - -```python -class msghandler: - DEFAULT_SIZE_THRESHOLD = 500_000 - - def __init__(self, broker_url=None, fileserver_url=None): - self.broker_url = broker_url or self.DEFAULT_BROKER_URL - self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL -``` - -### Dart Architecture - -Dart uses classes for stateful operations with async/await: - -- **Class-based msghandler**: Encapsulated API -- **Data classes**: Structured data (MsgPayloadV1, MsgEnvelopeV1) -- **Async/await**: I/O operations -- **dart-arrow**: Arrow IPC support (Desktop/Flutter only) -- **HTTP package**: HTTP file server communication -- **Transport package**: Transport client with WebSocket support (Dart Web) - -```dart -class msghandler { - static const DEFAULT_SIZE_THRESHOLD = 500000; - - final String brokerUrl; - final String fileserverUrl; - - msghandler({ - this.brokerUrl = DEFAULT_BROKER_URL, - this.fileserverUrl = 'http://localhost:8080', - }); -} -``` - -#### Dart Desktop (Dart SDK) - -- **Transport connections**: Uses broker URLs (e.g., `nats://`, `mqtt://`) -- **Apache Arrow IPC**: Full support via `dart-arrow` -- **Uint8List for binary data**: Native Dart binary handling - -#### Dart Flutter (Dart SDK) - -- **Transport connections**: Uses broker URLs (e.g., `nats://`, `mqtt://`) -- **Apache Arrow IPC**: Full support via `dart-arrow` -- **Uint8List for binary data**: Native Dart binary handling - -#### Dart Web (Dart SDK) - -- **WebSocket connections**: Uses `ws://` or `wss://` URLs (transport-agnostic) -- **No Apache Arrow**: Uses `jsontable` for tabular data only -- **Uint8List for binary data**: Browser-compatible binary handling -- **Fetch API**: HTTP file server communication via `http` package - -### Browser Architecture - -Browser JavaScript has specific constraints due to security and compatibility: - -- **Async/await**: Native async/await support -- **No Apache Arrow**: Arrow IPC not available in browsers -- **JSON table only**: Use "jsontable" for tabular data -- **WebSocket transport**: Uses transport client for browser-compatible connections -- **Fetch API**: HTTP file server communication via fetch - -### MicroPython Architecture - -MicroPython has significant constraints: - -- **Synchronous API**: No async/await -- **Memory-constrained**: 256KB - 1MB -- **Limited payload support**: No tables, max 50KB -- **Simplified UUID generation**: Custom implementation - -```python -# MicroPython constraints -DEFAULT_SIZE_THRESHOLD = 100_000 # 100KB -MAX_PAYLOAD_SIZE = 50_000 # 50KB hard limit -``` - -### Rust Architecture - -Rust leverages compile-time type safety and async runtimes: - -- **Type-safe payloads**: Rust enum discriminates between `Text`, `Dictionary`, `ArrowTable`, `Binary`, etc. -- **serde serialization**: Automatic JSON deserialization via `#[derive(Serialize, Deserialize)]` -- **tokio runtime**: Efficient async I/O for transport connections and HTTP file server operations -- **arrow2 integration**: Native Arrow IPC deserialization without intermediate format conversion -- **reqwest**: High-performance HTTP client with built-in TLS and connection pooling -- **Zero-copy patterns**: `Vec` passed directly to avoid unnecessary memory copies -- **Result**: Idiomatic error handling with typed error types - -```rust -// Type-safe payload enum (compile-time discrimination) -#[derive(Serialize, Deserialize, Clone)] -pub enum Payload { - Text(String), - Dictionary(serde_json::Value), - ArrowTable(Vec), - JsonTable(serde_json::Value), - Image(Vec), - Audio(Vec), - Video(Vec), - Binary(Vec), -} - -// Configuration via builder pattern -pub struct smartpackOptions { - pub broker_url: String, - pub fileserver_url: String, - pub fileserver_upload_handler: Option>, - pub size_threshold: usize, - pub correlation_id: String, - pub msg_purpose: String, - pub sender_name: String, - // ... other fields -} - -// Transport client with tokio integration -let conn = transport_client::connect(DEFAULT_BROKER_URL).await?; - -// Subscribe and process messages -let mut sub = conn.subscribe("/agent/wine/api/v1/analyze")?; -for msg in sub.messages() { - let envelope = smartunpack(&String::from_utf8_lossy(&msg.payload), &Default::default()).await?; - // Access deserialized payloads by type - for payload in &envelope.payloads { - match payload.payload_type.as_str() { - "arrowtable" => { /* payload.data is base64-encoded Arrow IPC */ }, - "text" => { /* payload.data is decoded text string */ }, - "binary" | "image" | "audio" | "video" => { /* payload.data is base64-encoded binary */ }, - _ => { /* other types */ } - } - } -} -``` - ---- - -## Scaling Architecture - -### Horizontal Scaling - -| Component | Scaling Strategy | -|-----------|------------------| -| **Message Broker** | Cluster deployment with multiple nodes | -| **File Server** | Load balancer + multiple instances | -| **Client Applications** | Deploy multiple instances behind load balancer | - -### Vertical Scaling - -| Component | Scaling Strategy | -|-----------|------------------| -| **Message Broker** | Increase memory, CPU, disk I/O | -| **File Server** | Increase memory, CPU, disk capacity | -| **Client Applications** | Increase heap size (Python/JS) | - -### Performance Considerations - -| Metric | Target | Notes | -|--------|--------|-------| -| Message serialization overhead | <50ms | For 10KB payload | -| Message deserialization overhead | <50ms | For 10KB payload | -| Transport connection establishment | <100ms | Connection pool recommended | -| File upload latency | <1s | For 0.5MB file | -| File download latency | <1s | For 0.5MB file | - ---- - -## Failure Modes and Recovery - -### Transport Connection Failure - -**Scenario**: Message broker unavailable - -**Handler**: -- Connection auto-reconnect via transport-level reconnection -- Retry with exponential backoff for publish operations - -**Recovery**: -- Transport client automatically attempts reconnection -- Application can check connection status before publishing - -### File Server Unavailable - -**Scenario**: HTTP file server unavailable during upload/download - -**Handler**: -- Retry up to 5 times with exponential backoff (100ms → 5000ms) -- Fallback to direct transport for upload (MicroPython) - -**Recovery**: -- Exponential backoff: `delay = min(delay * 2, max_delay)` -- After max retries, throw error with correlation ID - -### Deserialization Error - -**Scenario**: Payload type mismatch or corrupted data - -**Handler**: -- Log correlation ID and throw error -- No retry (data corruption) - -**Recovery**: -- Application must validate payload_type matches data type -- Use proper serialization before sending - -### Memory Overflow (MicroPython) - -**Scenario**: Payload exceeds maximum size (50KB) - -**Handler**: -- Reject payloads >50KB with MemoryError -- No retry (client-side check) - -**Recovery**: -- Application must split large payloads -- Use direct transport only for small payloads - ---- - -## Trade-off Decisions - -### Decision 1: Direct vs Link Transport Threshold - -**Trade-off**: Memory vs Network I/O - -**Decision**: Use 0.5MB threshold for desktop, 100KB for MicroPython - -**Rationale**: -- Direct transport uses more memory (Base64 encoding adds ~33% overhead) -- Link transport requires network I/O for upload/download -- 0.5MB is reasonable for desktop memory constraints -- 100KB is necessary for MicroPython memory constraints - -### Decision 2: Base64 Encoding for Direct Transport - -**Trade-off**: Bandwidth vs Simplicity - -**Decision**: Use Base64 encoding for all direct transport payloads - -**Rationale**: -- Simplifies JSON serialization (all data is string-compatible) -- Increases payload size by ~33%, but transport can handle this -- Alternative would be binary payload support (more complex) - -### Decision 3: Multiple Platform Implementations - -**Trade-off**: Development effort vs Cross-platform support - -**Decision**: Maintain separate implementations for each platform - -**Rationale**: -- Each platform has idiomatic patterns (multiple dispatch, async/await, etc.) -- Maintains developer productivity and code quality -- API parity ensures cross-platform compatibility - -### Decision 4: Handler Function Abstraction - -**Trade-off**: Flexibility vs Simplicity - -**Decision**: Abstract file server operations through handler functions - -**Rationale**: -- Allows support for different file server implementations (Plik, AWS S3, custom) -- Maintains simplicity for common use cases -- Enables plug-in architecture for custom backends - ---- - -## Deployment Architecture - -### Minimum Infrastructure - -| Component | Minimum | Notes | -|-----------|---------|-------| -| Message Broker | 1 instance | Single node for development | -| File Server | 1 instance | HTTP server for large payloads | -| Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) | -| Client Memory | 256KB | MicroPython devices | - -### Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `BROKER_URL` | `ws://localhost:4222` | Message broker URL | -| `FILESERVER_URL` | `http://localhost:8080` | HTTP file server URL | -| `SIZE_THRESHOLD` | `500000` | Size threshold in bytes (0.5MB) | - -### Container Deployment - -```mermaid -flowchart TD - subgraph "Docker Network" - Broker_Container[Message Broker] - FileServer_Container[Plik File Server] - App_Container[Application Container] - end - - App_Container -->|Transport| Broker_Container - App_Container -->|HTTP| FileServer_Container - - style Broker_Container fill:#fff3e0,stroke:#f57c00 - style FileServer_Container fill:#f3e5f5,stroke:#9c27b4 - style App_Container fill:#e3f2fd,stroke:#2196f3 -``` - ---- - -## Security Considerations - -### Payload Integrity - -**Mechanism**: SHA-256 checksum via metadata - -**Implementation**: -- Sender calculates checksum and stores in payload metadata -- Receiver validates checksum on receipt - -### Transport Security - -**Mechanism**: TLS support for transport connections - -**Implementation**: -- Use `nats://` URL for plain text -- Use `tls://` URL for TLS-encrypted connections -- Use `ws://` or `wss://` for WebSocket connections - -### File Server Security - -**Mechanism**: Authentication token for file uploads - -**Implementation**: -- Plik uses upload token in `X-UploadToken` header -- Application can implement custom authentication - ---- - -## Testing Architecture - -### Unit Test Coverage - -| Test Category | Coverage | Files | -|---------------|----------|-------| -| Serialization | All payload types | `test/test_*_sender.*` | -| Deserialization | All payload types | `test/test_*_receiver.*` | -| Transport selection | Direct vs link | `test/test_*_mix_payloads.*` | -| File server upload | Plik integration | Platform-specific | -| File server download | Exponential backoff | Platform-specific | - -### Integration Test Scenarios - -| Scenario | Platforms | Payloads | Transport | Expected Result | -|----------|-----------|----------|-----------|-----------------| -| Cross-platform text | Julia ↔ JS ↔ Python | text | direct | Round-trip successful | -| Arrow IPC round-trip | Julia ↔ JS ↔ Python | arrowtable | direct | Arrow IPC preserved | -| Large file transfer | All | image/audio/video | link | File server upload/download | -| Multi-payload mixed | All | text + image + file | direct/link | All payloads preserved | - ---- - -## Versioning - -### Architecture Versioning - -| Component | Version | Notes | -|-----------|---------|-------| -| Architecture | 1.0.0 | Initial release | -| Protocol | v1 | Message envelope protocol version | - -### Backward Compatibility - -| Version | Supported Platforms | -|---------|---------------------| -| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, Dart 2.17+, Rust 1.70+, MicroPython 1.19+ | - ---- - -## Change Log - -| Date | Version | Changes | -|------|---------|---------| -| 2026-05-15 | 1.5.0 | Made transport layer agnostic | All sections | -| - | - | Removed all NATS-specific references from architecture docs | All sections | -| - | - | Updated diagrams to use generic "Message Broker" instead of "NATS Server" | All sections | -| - | - | Updated code examples to use transport-agnostic patterns | All sections | -| - | - | Removed NATS client packages from external dependencies | All sections | -| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | -| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | -| - | - | Added `plik_upload_file` convenience function to component table | specification.md:13 | -| - | - | Fixed Rust payload access pattern (data is String, not Payload enum) | All sections | -| - | - | Fixed `smartpackOptions.fileserver_upload_handler` type to `Arc` | specification.md:13 | -| - | - | Removed `metadata` from link transport examples (now `None`/omitted) | specification.md:3 | -| - | - | Removed duplicate footer text | All sections | -| 2026-05-13 | 1.3.0 | Added Rust support with tokio, serde, and arrow2 | All sections | -| - | - | Added Rust to C4 diagrams (context, container) | All sections | -| - | - | Added Rust platform-specific architecture section | specification.md:13 | -| - | - | Updated component table with Rust support | All sections | -| 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | -| - | - | Removed publish_message component (commented out in source) | -| - | - | Removed NATSClient and NATSConnectionPool classes (not in ground truth) | -| - | - | Updated smartpack to return JSON for caller to publish via transport | -| - | - | Updated component diagram to match actual module structure | -| - | - | Updated data flow to show smartpack returns JSON for caller to publish | -| - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | -| 2026-03-15 | 1.1.0 | JavaScript connection management | -| - | - | Added NATSClient with keepAlive support | -| - | - | Added NATSConnectionPool for connection reuse | -| - | - | Added publishMessage function with closeConnection option | -| (Historical - pre-transport-agnostic refactor) | | | -| 2026-03-13 | 1.0.0 | Initial architecture documentation | - ---- - -## 16. References - -### 16.1 Documentation Artifacts - -| Document | Purpose | Specification Traceability | UI Specification Traceability | Requirement ID(s) | -|----------|---------|---------------------------|------------------------------|-------------------| -| [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 | - | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/specification.md`](./specification.md) | Technical contract for msghandler | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | - | All UI components and interactions | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/validation.md`](./validation.md) | CI/CD validation rules | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/runbook.md`](./runbook.md) | Operational runbook | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 | - -### 16.2 Implementation Files - -| File | Platform | Features | Specification Traceability | Requirement ID(s) | -|------|----------|----------|---------------------------|-------------------| -| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe, file upload helpers | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | specification.md:2-19 (all sections) | FR-005, FR-006, FR-012 | - -### 16.3 External Dependencies - -| Platform | Package | Version | Purpose | Specification Traceability | Requirement ID(s) | -|----------|---------|---------|---------|--------------------------|-------------------| -| Julia | JSON.jl | Latest | JSON serialization | specification.md:11 | FR-012, NFR-101, NFR-102 | -| Julia | Arrow.jl | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | -| Julia | HTTP.jl | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | -| Julia | UUIDs.jl | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 | -| Node.js | node-fetch | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | -| Browser | - | - | Transport-agnostic (caller provides) | specification.md:11 | FR-013, FR-014 | -| Python | aiohttp | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | -| Python | pyarrow | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | -| Dart | http | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | -| Dart | uuid | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 | -| Dart | dart-arrow | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | -| Rust | serde | Latest | JSON serialization | specification.md:11 | FR-012, NFR-101, NFR-102 | -| Rust | serde_json | Latest | JSON handling | specification.md:11 | FR-012, NFR-101, NFR-102 | -| Rust | tokio | Latest | Async runtime | specification.md:11 | FR-013, FR-014 | -| Rust | reqwest | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 | -| Rust | uuid | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 | -| Rust | arrow2 | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 | -| MicroPython | builtin | N/A | Limited implementation | specification.md:11 | FR-005, FR-006, FR-012 | - ---- - -## 17. Change Log - -| Date | Version | Changes | Specification Reference | -|------|---------|---------|------------------------| -| 2026-03-23 | 1.1.0 | Updated to ASG Framework architecture guidelines | specification.md:2-19 (all sections) | -| 2026-03-15 | 1.1.0 | JavaScript connection management | specification.md:2-19 (all sections) | -| 2026-03-13 | 1.0.0 | Initial architecture documentation | specification.md:2-19 (all sections) | - ---- - -## 18. Gap-Check Validation - -| Stage Transition | Gap-Check Question | Status | -|------------------|-------------------|--------| -| Requirements → Specification | Does the Specification define all edge cases and conflict scenarios from the Requirements? | ✅ Verified - All FR-XXX requirements have corresponding spec rules | -| Specification → UI Specification | Does the UI Specification expose all the data and states defined in the Specification? | ⏳ Pending - UI spec not yet created | -| UI Specification → Walkthrough | Does the Walkthrough reflect the complete flow including error states and timing? | ⏳ Pending - UI spec not yet created | -| Walkthrough → Architecture | Does the Architecture support the performance and integration requirements defined in the Walkthrough? | ✅ Verified - Architecture supports all walkthrough flows | - ---- - -*This architecture document is versioned and maintained in git alongside the codebase. All implementations must adhere to this architecture.* diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md new file mode 100644 index 0000000..77c21e3 --- /dev/null +++ b/docs/implementation-plan.md @@ -0,0 +1,416 @@ +# Implementation Plan: msghandler + +**Version**: 1.3.0 +**Date**: 2026-05-19 +**Status**: Active +**Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) + +--- + +## 1. Implementation Phases and Timeline + +### Phase 1: Core API Implementation (Week 1-2) +| Task | Priority | Estimated Effort | Status | +|------|----------|-----------------|--------| +| Core `smartpack()` implementation | P0 | 3 days | ✅ Complete | +| Core `smartunpack()` implementation | P0 | 3 days | ✅ Complete | +| Message envelope structure | P0 | 2 days | ✅ Complete | +| Payload type handling | P0 | 2 days | ✅ Complete | +| Transport adapter layer | P0 | 3 days | ✅ Complete | + +**Deliverables**: +- Julia module: `src/msghandler.jl` +- Node.js module: `src/msghandler_ssr.js` +- Browser module: `src/msghandler_csr.js` +- Python module: `src/msghandler.py` +- MicroPython module: `src/msghandler_mpy.py` + +### Phase 2: File Server Integration (Week 3) +| Task | Priority | Estimated Effort | Status | +|------|----------|-----------------|--------| +| File server upload handler | P1 | 2 days | ✅ Complete | +| File server download handler | P1 | 2 days | ✅ Complete | +| Exponential backoff logic | P1 | 1 day | ✅ Complete | +| Plik integration | P1 | 2 days | ✅ Complete | + +**Deliverables**: +- Upload handler with plik_oneshot_upload +- Download handler with retry logic +- Configurable file server URL + +### Phase 3: Platform-Specific Features (Week 4) +| Task | Priority | Estimated Effort | Status | +|------|----------|-----------------|--------| +| Arrow IPC support (Desktop) | P1 | 3 days | ✅ Complete | +| JSON table support (Browser) | P1 | 2 days | ✅ Complete | +| Browser WebSocket transport | P1 | 2 days | ✅ Complete | +| MicroPython optimizations | P2 | 2 days | ✅ Complete | + +**Deliverables**: +- Arrow IPC serialization for tabular data +- JSON table format for browser compatibility +- Browser-specific transport layer +- Memory-optimized MicroPython implementation + +### Phase 4: Cross-Platform Testing (Week 5) +| Task | Priority | Estimated Effort | Status | +|------|----------|-----------------|--------| +| Text message tests | P1 | 1 day | ✅ Complete | +| Dictionary tests | P1 | 1 day | ✅ Complete | +| Tabular data tests | P1 | 2 days | ✅ Complete | +| Mixed payload tests | P1 | 2 days | ✅ Complete | +| Large file tests | P1 | 2 days | ✅ Complete | + +**Deliverables**: +- Platform-specific test suites +- Integration test scenarios +- Performance benchmarks + +### Phase 5: Documentation & Examples (Week 6) +| Task | Priority | Estimated Effort | Status | +|------|----------|-----------------|--------| +| API documentation | P2 | 2 days | ✅ Complete | +| Walkthrough examples | P2 | 2 days | ✅ Complete | +| Architecture diagrams | P2 | 1 day | ✅ Complete | +| Deployment guides | P2 | 1 day | ✅ Complete | + +**Deliverables**: +- Comprehensive documentation +- Code examples for all platforms +- Deployment runbooks + +--- + +## 2. Module/Component Breakdown + +### Core Modules + +#### msghandler.jl (Julia) +``` +src/ +└── msghandler.jl + ├── Constants (DEFAULT_SIZE_THRESHOLD, etc.) + ├── msg_payload_v1 struct + ├── msg_envelope_v1 struct + ├── Serialization functions + │ ├── serialize_text() + │ ├── serialize_dictionary() + │ ├── serialize_arrowtable() + │ ├── serialize_jsontable() + │ └── serialize_binary() + ├── Deserialization functions + │ ├── deserialize_text() + │ ├── deserialize_dictionary() + │ ├── deserialize_arrowtable() + │ ├── deserialize_jsontable() + │ └── deserialize_binary() + ├── File server handlers + │ ├── plik_oneshot_upload() + │ └── _fetch_with_backoff() + ├── smartpack() - Main sender function + └── smartunpack() - Main receiver function +``` + +**Dependencies**: +- JSON.jl (JSON serialization) +- Arrow.jl (Arrow IPC) +- HTTP.jl (File server) +- UUIDs.jl (IDs) +- DataFrames.jl (DataFrame support) + +#### msghandler_ssr.js (Node.js) +``` +src/ +├── msghandler_ssr.js +│ ├── Constants +│ ├── msg_payload_v1 class +│ ├── msg_envelope_v1 class +│ ├── Serialization methods +│ ├── Deserialization methods +│ ├── File server handlers +│ ├── smartpack() function +│ └── smartunpack() function +└── nats/ + ├── NATSClient.js + └── NATSConnectionPool.js +``` + +**Dependencies**: +- nats (NATS client) +- node-fetch (HTTP file server) + +#### msghandler_csr.js (Browser) +``` +src/ +└── msghandler_csr.js + ├── Constants + ├── msg_payload_v1 class + ├── msg_envelope_v1 class + ├── Serialization methods (JSON table only) + ├── Deserialization methods + ├── File server handlers (browser-compatible) + ├── smartpack() function + └── smartunpack() function +``` + +**Dependencies**: +- nats.ws (Browser NATS client) + +#### msghandler.py (Python) +``` +src/ +└── msghandler.py + ├── Constants + ├── msg_payload_v1 class + ├── msg_envelope_v1 class + ├── Serialization methods + ├── Deserialization methods + ├── File server handlers + ├── smartpack() async function + └── smartunpack() async function +``` + +**Dependencies**: +- aiohttp (HTTP file server) +- pyarrow (Arrow IPC) +- uuid (IDs) + +#### msghandler.rs (Rust) +``` +src/ +├── msghandler.rs +│ ├── Constants +│ ├── msg_payload_v1 struct +│ ├── msg_envelope_v1 struct +│ ├── Serialization traits +│ ├── Deserialization traits +│ ├── File server handlers +│ ├── smartpack() async function +│ └── smartunpack() async function +├── Payload enum +├── smartpackOptions struct +└── smartunpackOptions struct +``` + +**Dependencies**: +- tokio (Async runtime) +- serde (JSON serialization) +- reqwest (HTTP file server) +- arrow2 (Arrow IPC) + +#### msghandler_mpy.py (MicroPython) +``` +src/ +└── msghandler_mpy.py + ├── Constants (lower thresholds) + ├── msg_payload_v1 class + ├── msg_envelope_v1 class + ├── serialize_text() + ├── deserialize_text() + ├── serialize_dictionary() + ├── deserialize_dictionary() + └── smartpack()/smartunpack() functions +``` + +**Constraints**: +- Limited to text and dictionary types +- Direct transport only (no file server) +- 100KB threshold for memory constraints + +--- + +## 3. Task List + +### Core API Tasks + +| Task ID | Description | Assignee | Priority | Status | +|---------|-------------|----------|----------|--------| +| T-001 | Implement `smartpack()` with tuple format | Developer A | P0 | ✅ Complete | +| T-002 | Implement `smartunpack()` with type handling | Developer A | P0 | ✅ Complete | +| T-003 | Create message envelope structure | Developer A | P0 | ✅ Complete | +| T-004 | Implement transport adapter | Developer B | P0 | ✅ Complete | +| T-005 | Add correlation ID support | Developer A | P0 | ✅ Complete | + +### File Server Tasks + +| Task ID | Description | Assignee | Priority | Status | +|---------|-------------|----------|----------|--------| +| T-006 | Implement Plik upload handler | Developer B | P1 | ✅ Complete | +| T-007 | Implement file download with retry | Developer B | P1 | ✅ Complete | +| T-008 | Add exponential backoff logic | Developer B | P1 | ✅ Complete | + +### Platform Tasks + +| Task ID | Description | Assignee | Priority | Status | +|---------|-------------|----------|----------|--------| +| T-009 | Implement Arrow IPC (Julia/Python/Node.js) | Developer A | P1 | ✅ Complete | +| T-010 | Implement JSON table (Browser) | Developer B | P1 | ✅ Complete | +| T-011 | Implement MicroPython optimizations | Developer C | P2 | ✅ Complete | +| T-012 | Browser WebSocket transport | Developer B | P1 | ✅ Complete | + +### Testing Tasks + +| Task ID | Description | Assignee | Priority | Status | +|---------|-------------|----------|------------------| +| T-013 | Text message tests | QA Team | P1 | ✅ Complete | +| T-014 | Dictionary tests | QA Team | P1 | ✅ Complete | +| T-015 | Tabular data tests | QA Team | P1 | ✅ Complete | +| T-016 | Mixed payload tests | QA Team | P1 | ✅ Complete | +| T-017 | Large file tests | QA Team | P1 | ✅ Complete | + +--- + +## 4. Test Strategy + +### Unit Tests + +| Test Category | Coverage | Files | Requirements | +|---------------|----------|-------|--------------| +| Serialization | All payload types | `test/test_*_sender.*` | FR-001 through FR-012 | +| Deserialization | All payload types | `test/test_*_receiver.*` | FR-001 through FR-012 | +| Transport selection | Direct vs link | `test/test_*_mix_payloads.*` | FR-003, FR-004, FR-006 | +| File server upload | Plik integration | Platform-specific | FR-008, FR-009 | +| File server download | Exponential backoff | Platform-specific | FR-010, FR-011 | + +### Integration Tests + +| Scenario | Platforms | Payloads | Transport | Requirements | +|----------|-----------|----------|-----------|--------------| +| Single text (small) | All | text | direct | FR-001, FR-012 | +| Single dictionary (small) | All | dictionary | direct | FR-002, FR-012 | +| Single arrow table (small) | Desktop | arrowtable | direct | FR-002, FR-012 | +| Single JSON table (small) | All | jsontable | direct | FR-001, FR-002, FR-006 | +| Single image (small) | All | image | direct | FR-001, FR-006 | +| Single text (large) | All | text | link | FR-003, FR-008, FR-009 | +| Mixed payloads | All | text + dictionary + image | mixed | FR-006, FR-007 | + +### Test Coverage Targets + +| Phase | Coverage Target | Method | +|-------|----------------|--------| +| Phase 1 | 70% | Unit tests per platform | +| Phase 2 | 80% | Add integration tests | +| Phase 3 | 85% | Add edge case tests | +| Phase 4 | 90% | Add performance tests | + +--- + +## 5. Build and Deployment Preparation + +### Continuous Integration + +| Check | Command | Purpose | +|-------|---------|---------| +| Linting | `npm run lint` | Code style enforcement | +| Type checking | `npx tsc --noEmit` | Type safety (JavaScript/TypeScript) | +| Unit tests | `npm test` | Functionality validation | +| Integration tests | `npm run test:integration` | Cross-platform validation | +| Coverage | `npm run coverage` | Test coverage tracking | + +### Deployment Pipeline + +``` +GitHub Push + ↓ +CI/CD Pipeline + ↓ +├──→ Linting (all platforms) +├──→ Unit tests (all platforms) +├──→ Integration tests (cross-platform) +├──→ Coverage report +└──→ Build documentation + ↓ +Release (if all checks pass) + ↓ +├──→ GitHub Releases +├──→ Package registry (npm, PyPI) +└──→ Documentation site +``` + +--- + +## 6. Risk Mitigation + +### Known Blockers + +| Risk | Mitigation Step | Owner | +|------|----------------|-------| +| **Browser Arrow IPC** | Use JSON table as fallback | Developer B | +| **MicroPython memory** | 100KB threshold, direct transport only | Developer C | +| **File server availability** | Exponential backoff with graceful degradation | Developer B | + +### Known Unknowns + +| Unknown | Monitoring Strategy | Response Plan | +|---------|-------------------|---------------| +| Platform-specific bugs | Comprehensive test coverage | Hotfix with platform-specific handling | +| Performance bottlenecks | Load testing and profiling | Optimized serialization/deserialization | + +--- + +## 7. Requirements Traceability + +### Functional Requirements + +| Requirement ID | Implementation Location | Status | +|---------------|------------------------|--------| +| FR-001 | All platform modules | ✅ Complete | +| FR-002 | All platform modules | ✅ Complete | +| FR-003 | All platform modules (size_threshold logic) | ✅ Complete | +| FR-004 | All platform modules | ✅ Complete | +| FR-005 | MicroPython module | ✅ Complete | +| FR-006 | All platform modules | ✅ Complete | +| FR-007 | All platform modules | ✅ Complete | +| FR-008 | All platform modules | ✅ Complete | +| FR-009 | All platform modules | ✅ Complete | +| FR-010 | All platform modules | ✅ Complete | +| FR-011 | All platform modules | ✅ Complete | +| FR-012 | All platform modules | ✅ Complete | +| FR-013 | All platform modules | ✅ Complete | +| FR-014 | All platform modules | ✅ Complete | + +### Non-Functional Requirements + +| NFR ID | Implementation Location | Status | +|--------|------------------------|--------| +| NFR-101 | Serialization functions | ✅ Complete | +| NFR-102 | Deserialization functions | ✅ Complete | +| NFR-103 | Transport adapter | ✅ Complete | +| NFR-104 | File upload handler | ✅ Complete | +| NFR-105 | File download handler | ✅ Complete | +| NFR-106 | MicroPython module | ✅ Complete | +| NFR-107 | Performance benchmarks | ✅ Complete | +| NFR-201 | Transport adapter | ✅ Complete | +| NFR-202 | File download retry logic | ✅ Complete | +| NFR-203 | Transport adapter | ✅ Complete | +| NFR-401 | Message envelope | ✅ Complete | +| NFR-402 | Metrics instrumentation | ✅ Complete | +| NFR-403 | Correlation ID propagation | ✅ Complete | + +--- + +## 8. Validation Gates + +### Pre-Release Checklist + +| Gate | Check | Pass Criteria | +|------|-------|--------------| +| **G-001** | All unit tests pass | 100% pass rate per platform | +| **G-002** | Integration tests pass | Cross-platform round-trip successful | +| **G-003** | Coverage threshold | ≥80% line coverage | +| **G-004** | Linting clean | No warnings or errors | +| **G-005** | Specification compliance | All spec rules validated | +| **G-006** | Documentation complete | All required docs present | + +### CI/CD Validation + +| Check | Command | Failure Action | +|-------|---------|---------------| +| Syntax | `julia --check-base` | Block PR | +| Unit tests | `julia test/runtests.jl` | Block PR | +| Integration | `npm run test:integration` | Block PR | +| Coverage | `codecov` | Report only | + +--- + +*This implementation plan is versioned and maintained in git alongside the codebase. All implementations must adhere to this plan.* diff --git a/docs/requirements.md b/docs/requirements.md index ce93f63..3c4af87 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -1,7 +1,7 @@ # Requirements Document: msghandler -**Version**: 1.2.0 -**Date**: 2026-05-13 +**Version**: 1.3.0 +**Date**: 2026-05-22 **Status**: Active **Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) @@ -33,16 +33,44 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | **As a developer**, I want automatic retry on file server download failures | P1 | Exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) | | **As a developer**, I want message tracing across distributed systems | P1 | Correlation ID is propagated through all message processing steps | -### 1.3 KPIs & Targets +### 1.3 Success Metrics & KPIs -| Metric | Target | Measurement Method | -|--------|--------|-------------------| -| 95% of messages complete within 200ms | 95% | Synthetic monitoring | -| <2 days from onboarding to first PR | 2 days | PR timeline tracking | -| 100% of messages validate against spec | 100% | CI block rate | -| >80% unit test coverage | 80% | Test coverage tools | -| <1% of PRs bypass validation gates | 1% | CI gate analysis | -| MTTR <15 minutes for P1 incidents | 15 minutes | Incident tracking | +**Functional Requirements KPIs:** +- **FR-001** (Cross-platform text messaging): 95% of text messages delivered correctly across all platform pairs (<200ms latency) - Measured via synthetic cross-platform tests +- **FR-002** (Cross-platform tabular data): 100% Arrow IPC round-trip integrity (Desktop), 100% JSON table round-trip integrity (Browser) - Measured via data validation tests +- **FR-003** (Large file handling): 99% successful file uploads to server for payloads ≥0.5MB - Measured via integration tests +- **FR-004** (Direct transport for small payloads): 100% of payloads <0.5MB use direct transport - Measured via transport selection tests +- **FR-005** (MicroPython support): 100% of payloads <100KB delivered on MicroPython devices - Measured via MicroPython integration tests +- **FR-006** (Multi-payload messages): 100% correct parsing of multi-payload message lists - Measured via multi-payload tests +- **FR-007** (Payload type preservation): 100% type integrity preserved across all platforms - Measured via type validation tests +- **FR-008** (Plik file server integration): 100% successful Plik upload/token handling - Measured via Plik integration tests +- **FR-009** (Custom file server support): 100% handler abstraction works with custom implementations - Measured via custom server integration tests +- **FR-010** (Exponential backoff retry): 95% successful downloads within retry limit - Measured via failure injection tests +- **FR-011** (Correlation ID propagation): 100% correlation IDs propagated through all steps - Measured via tracing tests +- **FR-012** (Message serialization): <50ms serialization overhead for 10KB payload - Measured via benchmark tests +- **FR-013** (Transport publishing): 100% JSON envelope generated correctly - Measured via serialization tests +- **FR-014** (Transport subscription): 100% JSON messages processed correctly - Measured via deserialization tests + +**Non-Functional Requirements KPIs:** +- **NFR-101** (Message serialization overhead): <50ms for 10KB payload - Measured via benchmark tests +- **NFR-102** (Message deserialization overhead): <50ms for 10KB payload - Measured via benchmark tests +- **NFR-103** (Transport connection establishment): <100ms average - Measured via connection pool benchmarks +- **NFR-104** (File upload latency): <1s for 0.5MB file - Measured via integration tests +- **NFR-105** (File download latency): <1s for 0.5MB file - Measured via integration tests +- **NFR-106** (Concurrent connections): 100+ simultaneous transport connections - Measured via scale testing +- **NFR-107** (Message throughput): 1000+ messages/second per instance - Measured via load testing +- **NFR-108** (File server scalability): Horizontal scaling verified via architecture review +- **NFR-201** (Message delivery): At-least-once delivery via transport - Measured via message acknowledgment tests +- **NFR-202** (File server availability): <5% failure rate when file server unavailable - Measured via failure injection tests +- **NFR-203** (Connection recovery): Auto-reconnect within 30s - Measured via connection failure tests +- **NFR-301** (Payload integrity): 100% SHA-256 checksum validation - Measured via integrity tests +- **NFR-302** (Transport security): 100% TLS connections in production - Measured via connection audits +- **NFR-303** (File server security): 100% authenticated file uploads - Measured via security tests +- **NFR-401** (Required logs): 100% messages logged with required fields - Measured via log validation +- **NFR-402** (Critical metrics): 100% metrics collected with 1-minute granularity - Measured via metrics pipeline tests +- **NFR-403** (Tracing): 100% correlation ID propagation for tracing - Measured via tracing validation +- **NFR-404** (Alerting): <5min alert latency for `download_retry_exceeded` - Measured via alert pipeline tests +- **NFR-405** (Retention): Logs: 30 days, Metrics: 1 year - Measured via storage audits --- @@ -108,63 +136,68 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | ID | Requirement | Description | |----|-------------|-------------| -| **FR-001** | Cross-platform text messaging | System shall allow users to send text messages between Julia, JavaScript, Python, and MicroPython applications | -| **FR-002** | Cross-platform tabular data | System shall support DataFrame exchange between Julia and Python applications using Arrow IPC format | -| **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via transport | -| **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via transport without file server upload | -| **FR-005** | MicroPython support | System shall support payloads <100KB on MicroPython devices using direct transport | -| **FR-006** | Multi-payload messages | System shall accept and process lists of (dataname, data, type) tuples | -| **FR-007** | Payload type preservation | System shall preserve payload types when returning multi-payload messages | -| **FR-008** | Plik file server integration | System shall support Plik one-shot upload mode with upload ID and token handling | -| **FR-009** | Custom file server support | System shall provide handler function abstraction for custom HTTP file server implementations | -| **FR-010** | Exponential backoff retry | System shall implement exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) for file server download failures | -| **FR-011** | Correlation ID propagation | System shall propagate correlation IDs through all message processing steps | -| **FR-012** | Message serialization | System shall serialize data types using Base64, JSON, or Arrow IPC encoding | -| **FR-013** | Transport publishing | System shall return JSON string representation for caller to publish via transport layer (caller is responsible for actual transport publish) | -| **FR-014** | Transport subscription | System shall receive and process messages by accepting JSON string from transport payload | +| **FR-001** | Cross-platform text messaging | System shall allow users to send text messages between Julia, JavaScript, Python, and MicroPython applications | FR-001 KPI: 95% of text messages delivered correctly across all platform pairs (<200ms latency) | +| **FR-002** | Cross-platform tabular data | System shall support DataFrame exchange between Julia and Python applications using Arrow IPC format | FR-002 KPI: 100% Arrow IPC round-trip integrity (Desktop), 100% JSON table round-trip integrity (Browser) | +| **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via transport | FR-003 KPI: 99% successful file uploads to server for payloads ≥0.5MB | +| **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via transport without file server upload | FR-004 KPI: 100% of payloads <0.5MB use direct transport | +| **FR-005** | MicroPython support | System shall support payloads <100KB on MicroPython devices using direct transport | FR-005 KPI: 100% of payloads <100KB delivered on MicroPython devices | +| **FR-006** | Multi-payload messages | System shall accept and process lists of (dataname, data, type) tuples | FR-006 KPI: 100% correct parsing of multi-payload message lists | +| **FR-007** | Payload type preservation | System shall preserve payload types when returning multi-payload messages | FR-007 KPI: 100% type integrity preserved across all platforms | +| **FR-008** | Plik file server integration | System shall support Plik one-shot upload mode with upload ID and token handling | FR-008 KPI: 100% successful Plik upload/token handling | +| **FR-009** | Custom file server support | System shall provide handler function abstraction for custom HTTP file server implementations | FR-009 KPI: 100% handler abstraction works with custom implementations | +| **FR-010** | Exponential backoff retry | System shall implement exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) for file server download failures | FR-010 KPI: 95% successful downloads within retry limit | +| **FR-011** | Correlation ID propagation | System shall propagate correlation IDs through all message processing steps | FR-011 KPI: 100% correlation IDs propagated through all steps | +| **FR-012** | Message serialization | System shall serialize data types using Base64, JSON, or Arrow IPC encoding | FR-012 KPI: <50ms serialization overhead for 10KB payload | +| **FR-013** | Transport publishing | System shall return JSON string representation for caller to publish via transport layer (caller is responsible for actual transport publish) | FR-013 KPI: 100% JSON envelope generated correctly | +| **FR-014** | Transport subscription | System shall receive and process messages by accepting JSON string from transport payload | FR-014 KPI: 100% JSON messages processed correctly | --- ## 4. Non-Functional Requirements (NFRs) +**Requirement vs KPI Clarification:** +- **FR and NFR** is a *requirement* — it defines what quality or constraint the system must have (e.g., "System shall support 10K TPS", "99.9% monthly uptime", "TLS 1.3+ encryption") +- **KPI** is a *measurement* — it's the actual data collected to verify if the requirement was met (e.g., "Peak traffic was 8.5K TPS", "MTTR was 8 minutes", "100% of connections use TLS 1.3") +- Requirements tell you **what to build**; KPIs tell you **how well you built it** + ### 4.1 Performance & Scalability -| ID | Requirement | Specification | Test Method | -|----|-------------|---------------|-------------| -| **NFR-101** | Message serialization overhead | <50ms for 10KB payload | Benchmark tests | -| **NFR-102** | Message deserialization overhead | <50ms for 10KB payload | Benchmark tests | -| **NFR-103** | Transport connection establishment | <100ms | Connection pool benchmarks | -| **NFR-104** | File upload latency | <1s for 0.5MB file | Integration tests | -| **NFR-105** | File download latency | <1s for 0.5MB file | Integration tests | -| **NFR-106** | Concurrent connections | Support 100+ simultaneous transport connections | Scale testing | -| **NFR-107** | Message throughput | Handle 1000+ messages/second per instance | Load testing | -| **NFR-108** | File server scalability | Support horizontal scaling of file server backend | Architecture review | +| ID | Requirement | Specification | KPI | Test Method | +|----|-------------|---------------|-----|-------------| +| **NFR-101** | Message serialization overhead | <50ms for 10KB payload | <50ms for 10KB payload | Benchmark tests | +| **NFR-102** | Message deserialization overhead | <50ms for 10KB payload | <50ms for 10KB payload | Benchmark tests | +| **NFR-103** | Transport connection establishment | <100ms | <100ms average | Connection pool benchmarks | +| **NFR-104** | File upload latency | <1s for 0.5MB file | <1s for 0.5MB file | Integration tests | +| **NFR-105** | File download latency | <1s for 0.5MB file | <1s for 0.5MB file | Integration tests | +| **NFR-106** | Concurrent connections | Support 100+ simultaneous transport connections | 100+ simultaneous connections | Scale testing | +| **NFR-107** | Message throughput | Handle 1000+ messages/second per instance | 1000+ messages/second | Load testing | +| **NFR-108** | File server scalability | Support horizontal scaling of file server backend | Horizontal scaling verified | Architecture review | ### 4.2 Availability & Reliability -| ID | Requirement | Specification | -|----|-------------|---------------| -| **NFR-201** | Message delivery | At-least-once delivery semantics via transport | -| **NFR-202** | File server availability | Graceful degradation when file server is unavailable | -| **NFR-203** | Connection recovery | Auto-reconnect on transport connection failure | +| ID | Requirement | Specification | KPI | Test Method | +|----|-------------|---------------|-----|-------------| +| **NFR-201** | Message delivery | At-least-once delivery semantics via transport | At-least-once delivery via transport | Message acknowledgment tests | +| **NFR-202** | File server availability | Graceful degradation when file server is unavailable | <5% failure rate when file server unavailable | Failure injection tests | +| **NFR-203** | Connection recovery | Auto-reconnect on transport connection failure | Auto-reconnect within 30s | Connection failure tests | ### 4.3 Privacy & Security -| ID | Requirement | Specification | -|----|-------------|---------------| -| **NFR-301** | Payload integrity | SHA-256 checksum support via metadata | -| **NFR-302** | Transport security | TLS support for transport connections | -| **NFR-303** | File server security | Authentication token for file uploads | +| ID | Requirement | Specification | KPI | Test Method | +|----|-------------|---------------|-----|-------------| +| **NFR-301** | Payload integrity | SHA-256 checksum support via metadata | 100% SHA-256 checksum validation | Integrity tests | +| **NFR-302** | Transport security | TLS support for transport connections | 100% TLS connections in production | Connection audits | +| **NFR-303** | File server security | Authentication token for file uploads | 100% authenticated file uploads | Security tests | ### 4.4 Observability & Telemetry -| ID | Requirement | Specification | -|----|-------------|---------------| -| **NFR-401** | Required logs | `correlation_id`, `msg_id`, `timestamp`, `sender_name`, `receiver_name`, `payload_type`, `transport` | -| **NFR-402** | Critical metrics | `messages_sent_total`, `messages_received_total`, `file_upload_duration_seconds`, `file_download_duration_seconds`, `retry_attempts_total` | -| **NFR-403** | Tracing | Correlation ID propagation for request tracing | -| **NFR-404** | Alerting | `download_retry_exceeded` triggers alert when max retries exceeded | -| **NFR-405** | Retention | Logs: 30 days, Metrics: 1 year | +| ID | Requirement | Specification | KPI | Test Method | +|----|-------------|---------------|-----|-------------| +| **NFR-401** | Required logs | `correlation_id`, `msg_id`, `timestamp`, `sender_name`, `receiver_name`, `payload_type`, `transport` | 100% messages logged with required fields | Log validation | +| **NFR-402** | Critical metrics | `messages_sent_total`, `messages_received_total`, `file_upload_duration_seconds`, `file_download_duration_seconds`, `retry_attempts_total` | 100% metrics collected with 1-minute granularity | Metrics pipeline tests | +| **NFR-403** | Tracing | Correlation ID propagation for request tracing | 100% correlation ID propagation for tracing | Tracing validation | +| **NFR-404** | Alerting | `download_retry_exceeded` triggers alert when max retries exceeded | <5min alert latency for `download_retry_exceeded` | Alert pipeline tests | +| **NFR-405** | Retention | Logs: 30 days, Metrics: 1 year | Logs: 30 days, Metrics: 1 year | Storage audits | --- @@ -173,7 +206,7 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | Condition | Description | |-----------|-------------| | **AC-001** | All functional requirements FR-001 through FR-014 are implemented and tested | -| **AC-002** | All non-functional requirements NFR-101 through NFR-405 meet specified targets | +| **AC-002** | All non-functional requirements NFR-101 through NFR-405 meet specified KPI targets | | **AC-003** | Cross-platform text message test passes (Julia ↔ JavaScript ↔ Python) | | **AC-004** | Cross-platform tabular data test passes with Arrow IPC round-trip (Desktop) | | **AC-005** | Cross-platform tabular data test passes with JSON table round-trip (Browser) | @@ -406,6 +439,10 @@ function smartunpack( | Date | Version | Changes | |------|---------|---------| +| 2026-05-22 | 1.3.0 | Updated to ASG Framework v8 pillars - added KPIs to all FR and NFR requirements | +| - | - | Added Success Metrics & KPIs section with measurable targets for each requirement | +| - | - | Added NFR vs KPI clarification section | +| - | - | Updated NFR tables to include KPI column and Test Method column | | 2026-05-15 | 1.3.0 | Made transport layer agnostic | | - | - | Removed all NATS-specific dependencies and references | | - | - | Updated all NATS references to generic "transport layer"/"message broker" | diff --git a/docs/solution-design.md b/docs/solution-design.md new file mode 100644 index 0000000..cc57b35 --- /dev/null +++ b/docs/solution-design.md @@ -0,0 +1,345 @@ +# Solution Design: msghandler + +**Version**: 1.3.0 +**Date**: 2026-05-22 +**Status**: Active +**Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) + +--- + +## 1. Problem Decomposition + +msghandler addresses the challenge of cross-platform data exchange between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using message brokers as transport layers. + +### Problem Statement + +Developers working across multiple programming languages face significant obstacles when trying to share data: + +| Problem | Description | User Impact | +|---------|-------------|-------------| +| **P-001**: Cross-platform data serialization | Different languages have incompatible data types and serialization formats | Developers must write platform-specific conversion code | +| **P-002**: Large payload handling | Message brokers have size limits, but large files need to be transferred | Large files either fail or require complex workarounds | +| **P-003**: Transport abstraction | Each platform has different message broker libraries and APIs | No unified interface across platforms | +| **P-004**: Request-response patterns | Bi-directional communication requires complex correlation tracking | Developers must implement custom message routing | + +### Solution Boundaries + +**In Scope**: +- Unified API for `smartpack()` and `smartunpack()` across all platforms +- Automatic transport selection based on payload size +- File server integration using Claim-Check pattern +- Multi-payload support with mixed types in single message +- Exponential backoff for reliable file downloads + +**Out of Scope**: +- Message compression (adds complexity without clear benefit) +- Message encryption (application-layer concern) +- Advanced message routing (simple topic matching sufficient) +- Persistent message queues (transport pattern sufficient) + +### Decision IDs + +| Decision ID | Decision | Description | +|-------------|----------|-------------| +| SD-001 | Claim-Check Pattern | Large payloads uploaded to HTTP server, small payloads sent directly | +| SD-002 | Automatic Transport Selection | <0.5MB = direct, ≥0.5MB = link based on size threshold | +| SD-003 | Handler Function Abstraction | Pluggable file server implementations via handler functions | +| SD-004 | Unified Tuple Format | Same `(dataname, data, type)` format across all platforms | +| SD-005 | Base64 Encoding | JSON-compatible binary data transport | +| SD-006 | Transport Abstraction | Support multiple broker protocols (NATS/MQTT/WebSocket) transparently | + +--- + +## 2. Solution Approach + +msghandler implements a **Claim-Check pattern** with intelligent transport selection: + +``` +Sender (smartpack) Transport Layer Receiver (smartunpack) +┌─────────────────┐ ┌───────────────┐ ┌───────────────────┐ +│ │ │ │ │ │ +│ 1. Data tuples │────────────>│ │───────────>│ 1. Parse envelope │ +│ [(name, │ JSON │ Message │ JSON │ 2. Check transport│ +│ data, type)]│ format │ Broker │ format │ 3. Fetch/Decode │ +│ │ │ (NATS/MQTT/ │ │ 4. Return tuples │ +└─────────────────┘ │ WebSocket) │ │ │ + │ │ └───────────────────┘ + └───────────────┘ +``` + +### Key Design Decisions + +| Decision ID | Decision | Rationale | Alternatives Rejected | +|-------------|----------|-----------|----------------------| +| **SD-001** | Claim-Check Pattern | Large payloads (>0.5MB) uploaded to HTTP server, small payloads sent directly via transport | Client-side compression - adds complexity; Server-side compression - not universally supported | +| **SD-002** | Automatic Transport Selection | <0.5MB = direct (fast), ≥0.5MB = link (avoid transport limits) | Manual selection - error-prone; Fixed threshold - not adaptive | +| **SD-003** | Handler Function Abstraction | Allows pluggable file server implementations (Plik, AWS S3, custom) | Hardcoded Plik - not flexible; Interface-based - too complex for this use case | +| **SD-004** | Unified Tuple Format | Same input/output format across all platforms | Platform-native formats - no interoperability; Protocol buffers - too heavy | +| **SD-005** | Base64 Encoding | JSON-compatible binary data transport | Raw bytes - not JSON-compatible; Hex encoding - 2x size overhead | +| **SD-006** | Transport Abstraction | Support multiple broker protocols (NATS/MQTT/WebSocket) transparently | Platform-specific libraries - no interoperability | + +### Architecture Components + +```mermaid +flowchart TB + subgraph Client["Client Application"] + direction TB + APP["Application Code"] + API["msghandler API"] + + APP -->|Data tuples| API + API -->|JSON envelope| TRANSPORT + end + + subgraph Transport["Transport Layer"] + direction TB + BROKER["Message Broker
NATS/MQTT/WebSocket"] + TOPICS["Topic Subscription"] + + API -->|Publish| BROKER + BROKER -->|Deliver| TOPICS + TOPICS -->|Subscribe| API + end + + subgraph FileServer["File Server"] + direction TB + UPLOAD["Upload Handler"] + DOWNLOAD["Download Handler"] + + API -.->|Upload URL| UPLOAD + DOWNLOAD -.->|Fetch URL| API + end + + style CLIENT fill:#e1f5fe,stroke:#0288d1,stroke-width:2px + style Transport fill:#ffe0b2,stroke:#f57c00,stroke-width:2px + style FileServer fill:#c8e6c9,stroke:#43a047,stroke-width:2px +``` + +--- + +## 3. Alternatives Considered + +| Alternative | Pros | Cons | Decision | +|-------------|------|------|----------| +| **gRPC/Protobuf** | Strong typing, efficient binary format | No native MicroPython support; Complex schema management | Rejected - not cross-platform enough | +| **MessagePack** | Compact binary, good performance | Browser support limited; No standard for tabular data | Rejected - missing Arrow IPC alternative | +| **Protocol Buffers** | Type-safe, efficient | No native support for tabular data exchange | Rejected - cannot represent DataFrames natively | +| **REST HTTP Upload** | Simple, universal | High latency; No real-time capability | Rejected - not suitable for message broker pattern | +| **Hybrid (direct/link)** | Optimal for both small and large payloads | More complex implementation | Accepted - matches user requirements (FR-003, FR-004) | +| **Single transport type** | Simpler implementation | Cannot handle large payloads efficiently | Rejected - violates FR-003 requirement | +| **Platform-specific APIs** | Native performance | No interoperability; Maintenance burden | Rejected - violates cross-platform goal | + +--- + +## 4. High-Level Component Diagram + +```mermaid +flowchart TD + subgraph msghandler["msghandler Core Module"] + direction TB + + subgraph Serialization["Serialization Layer"] + DIR["Direct Transport"] + LNK["Link Transport"] + + DIR -->|Base64| JSON_MSG + LNK -->|HTTP URL| JSON_MSG + end + + subgraph Envelope["Envelope Builder"] + HDR["Message Header"] + PAY["Payload Manager"] + + HDR --> PAY + end + + subgraph Handlers["Handler Functions"] + UPD["Upload Handler"] + DWN["Download Handler"] + + UPD --> LNK + DWN --> LNK + end + + API["smartpack() / smartunpack()"] + + API -->|Input| Serialization + API -->|Output| Serialization + API -->|Configure| Handlers + end + + subgraph Transport["Transport Layer"] + BROKER["NATS / MQTT / WebSocket"] + API -->|JSON| BROKER + BROKER -->|JSON| API + end + + subgraph FileServer["File Server"] + Plik["HTTP Server"] + UPD -.->|POST| Plik + Plik -.->|URL| DWN + end + + style msghandler fill:#b3e5fc,stroke:#0288d1,stroke-width:2px + style Transport fill:#ffe0b2,stroke:#f57c00,stroke-width:2px + style FileServer fill:#c8e6c9,stroke:#43a047,stroke-width:2px +``` + +### Component Responsibilities + +| Component | Responsibilities | Decision IDs | Requirements Addressed | +|-----------|-----------------|--------------|----------------------| +| **Serialization Layer** | Convert data types to transport format (Base64/URL) | SD-005 | FR-001, FR-002, FR-012 | +| **Envelope Builder** | Create standardized message envelope with metadata | SD-001 | FR-011, FR-013, FR-014 | +| **Handler Functions** | Abstract file server operations for pluggability | SD-003 | FR-008, FR-009 | +| **Transport Adapter** | Support multiple broker protocols transparently | SD-006 | FR-013, FR-014 | +| **Payload Manager** | Track payload types, sizes, and encoding | SD-004 | FR-006, FR-007 | + +--- + +## 5. Decision Rationale + +### SD-001: Why Claim-Check Pattern? + +**Requirement**: FR-003 - Large file handling, FR-004 - Direct transport for small payloads + +**Rationale**: +- Transport layers (NATS, MQTT) have message size limits (typically 1MB) +- Direct transport is faster for small payloads (no file server round-trip) +- Link transport avoids transport limits for large payloads +- User doesn't need to manually choose - automatic selection based on threshold + +### SD-002: Why Handler Functions for File Server? + +**Requirement**: FR-008 - Plik integration, FR-009 - Custom file server support + +**Rationale**: +- Plik is common open-source solution for file server +- Some users need AWS S3 or custom implementation +- Handler functions provide clean abstraction without vendor lock-in +- Same signature across all platforms (unified API) + +### SD-003: Why Tuple Format for Payloads? + +**Requirement**: FR-006 - Multi-payload messages, FR-007 - Payload type preservation + +**Rationale**: +- `(dataname, data, type)` tuple is language-agnostic +- Simple to understand: name, content, type +- Supports mixed payload types in single message +- Easy to serialize/deserialize across platforms + +### SD-004: Why Base64 Encoding? + +**Requirement**: FR-012 - Message serialization, FR-001 - Cross-platform text messaging + +**Rationale**: +- JSON is universal - works on all platforms +- Base64 converts binary to ASCII for JSON compatibility +- Standard format with native support in all languages +- No additional dependencies needed + +### SD-005: Why Automatic Transport Selection? + +**Requirement**: FR-003, FR-004, NFR-104, NFR-105 + +**Rationale**: +- <0.5MB payloads use direct transport (<1s latency, FR-004 KPI) +- ≥0.5MB payloads use link transport to avoid transport limits (FR-003 KPI: 99% successful uploads) +- User doesn't need to manually choose - automatic selection based on threshold + +### SD-006: Why Transport Abstraction? + +**Requirement**: FR-013, FR-014, NFR-201 + +**Rationale**: +- Support multiple broker protocols (NATS, MQTT, WebSocket) transparently +- Caller handles actual transport publishing/subscription +- Unified API across all platforms +- At-least-once delivery semantics via transport layer + +--- + +## 6. Risk Assessment + +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| **Performance degradation with >500KB payloads** | High | Medium | Size threshold detection; Link transport fallback | +| **File server availability issues** | Medium | Low | Exponential backoff retry; Graceful degradation | +| **Platform-specific bugs** | Medium | Low | Comprehensive test suite per platform; CI validation | +| **Encoding mismatches between platforms** | High | Low | Strict specification; Test contracts; Validation rules | +| **Transport layer incompatibility** | Medium | Low | Transport-agnostic design; Handler abstraction | + +--- + +## 7. Requirements Traceability + +| Solution Component | Decision ID | Requirement ID | Description | +|-------------------|-------------|----------------|-------------| +| **smartpack() function** | SD-001, SD-002, SD-004, SD-005, SD-006 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unified API for sending messages across all platforms | +| **smartunpack() function** | SD-001, SD-002, SD-004, SD-005, SD-006 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unified API for receiving messages across all platforms | +| **Direct transport** | SD-002 | FR-004, NFR-101, NFR-102, NFR-104, NFR-105 | Send payloads < threshold directly via transport | +| **Link transport** | SD-001, SD-002 | FR-003, NFR-104, NFR-105 | Upload payloads ≥ threshold to file server | +| **File server handler** | SD-003 | FR-008, FR-009, FR-010 | Pluggable upload/download handlers with retry logic | +| **Payload type preservation** | SD-004 | FR-006, FR-007 | Support text, dictionary, arrowtable, jsontable, image, audio, video, binary | +| **Correlation ID** | SD-001 | FR-011, NFR-401, NFR-403 | Message tracing across distributed systems | +| **Multi-payload support** | SD-004 | FR-006, FR-007 | List of (dataname, data, type) tuples | + +### Non-Functional Requirements Traceability + +| Solution Component | Decision ID | NFR ID | Description | +|-------------------|-------------|--------|-------------| +| **Serialization optimization** | SD-005 | NFR-101, NFR-102 | <50ms overhead for 10KB payloads | +| **Transport efficiency** | SD-006 | NFR-103 | <100ms connection establishment | +| **File server latency** | SD-001, SD-002 | NFR-104, NFR-105 | <1s upload/download for 0.5MB files | +| **Concurrent connections** | SD-006 | NFR-106 | Support 100+ simultaneous connections | +| **Message throughput** | SD-005, SD-006 | NFR-107 | Handle 1000+ messages/second per instance | +| **At-least-once delivery** | SD-006 | NFR-201 | Transport layer semantics | +| **Graceful degradation** | SD-003 | NFR-202 | File server unavailability handling | +| **Auto-reconnect** | SD-006 | NFR-203 | Transport connection failure recovery | +| **Required logs** | SD-001 | NFR-401 | Correlation ID, msg_id, timestamp, etc. | +| **Critical metrics** | SD-001, SD-005 | NFR-402 | messages_sent_total, file upload/download duration | +| **Tracing** | SD-001 | NFR-403 | Correlation ID propagation | + +--- + +## 8. Gap-Check Validation + +| Stage Transition | Gap-Check Question | Status | +|------------------|-------------------|--------| +| **Requirements → Solution Design** | Does the Solution Design clearly explain how the system solves the user problem, not just what it does? | ✅ Verified - All user stories mapped to solution components with requirement ID and decision ID references | +| **Solution Design → Specification** | Does the Specification define all technical details that the solution approach requires? | ⏳ Pending - Specification needs review for completeness | +| **Solution Design → Walkthrough** | Does the Walkthrough reflect the complete flow including error states and timing? | ⏳ Pending - Walkthrough needs validation against design | + +### Solution Design Validation + +**Problem**: Users need to send mixed payload types (text + image + large file) between Julia, JavaScript, Python, and MicroPython applications. + +**Solution Components**: +1. **SD-001** - `smartpack()` - Unified API for all platforms +2. **SD-002** - Tuple format - `(dataname, data, type)` - platform-agnostic +3. **SD-003** - Automatic transport selection - <0.5MB = direct, ≥0.5MB = link +4. **SD-004** - File server handler abstraction - Plik/AWS S3/custom support +5. **SD-005** - Exponential backoff - Reliable file downloads +6. **SD-006** - Correlation ID - Message tracing + +**Requirement Mapping**: +- FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 ✅ + +**Gap Check**: Does this solution explain *how* users will actually use the system? + +**Answer**: Yes - the walkthrough provides concrete examples: +1. JavaScript sends `[(msg, "Hello", "text"), (avatar, binary_data, "image")]` +2. `smartpack()` automatically selects transport based on size (SD-002) +3. Large file (≥0.5MB) → link transport → file server upload (SD-001) +4. Small payload (<0.5MB) → direct transport → base64 encoding (SD-005) +5. Receiver calls `smartunpack()` → receives same tuple format + +--- + +*This solution design document is versioned and maintained in git alongside the codebase. All implementations must adhere to this design.* + +**Traceability Summary**: +- All requirements traced to solution components with SD-XXX decision IDs +- Each decision ID references the corresponding requirement IDs (FR-XXX, NFR-XXX) +- Specification must cite SD-XXX references for each technical detail From 72e0c3be1e9c609e8f42f572a9ee869e69f2a3c5 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 07:07:06 +0700 Subject: [PATCH 06/14] update requirement doc --- docs/requirements.md | 70 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/docs/requirements.md b/docs/requirements.md index 3c4af87..70abc0d 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -136,20 +136,20 @@ msghandler is a cross-platform, bi-directional data bridge that enables seamless | ID | Requirement | Description | |----|-------------|-------------| -| **FR-001** | Cross-platform text messaging | System shall allow users to send text messages between Julia, JavaScript, Python, and MicroPython applications | FR-001 KPI: 95% of text messages delivered correctly across all platform pairs (<200ms latency) | -| **FR-002** | Cross-platform tabular data | System shall support DataFrame exchange between Julia and Python applications using Arrow IPC format | FR-002 KPI: 100% Arrow IPC round-trip integrity (Desktop), 100% JSON table round-trip integrity (Browser) | -| **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via transport | FR-003 KPI: 99% successful file uploads to server for payloads ≥0.5MB | -| **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via transport without file server upload | FR-004 KPI: 100% of payloads <0.5MB use direct transport | -| **FR-005** | MicroPython support | System shall support payloads <100KB on MicroPython devices using direct transport | FR-005 KPI: 100% of payloads <100KB delivered on MicroPython devices | -| **FR-006** | Multi-payload messages | System shall accept and process lists of (dataname, data, type) tuples | FR-006 KPI: 100% correct parsing of multi-payload message lists | -| **FR-007** | Payload type preservation | System shall preserve payload types when returning multi-payload messages | FR-007 KPI: 100% type integrity preserved across all platforms | -| **FR-008** | Plik file server integration | System shall support Plik one-shot upload mode with upload ID and token handling | FR-008 KPI: 100% successful Plik upload/token handling | -| **FR-009** | Custom file server support | System shall provide handler function abstraction for custom HTTP file server implementations | FR-009 KPI: 100% handler abstraction works with custom implementations | -| **FR-010** | Exponential backoff retry | System shall implement exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) for file server download failures | FR-010 KPI: 95% successful downloads within retry limit | -| **FR-011** | Correlation ID propagation | System shall propagate correlation IDs through all message processing steps | FR-011 KPI: 100% correlation IDs propagated through all steps | -| **FR-012** | Message serialization | System shall serialize data types using Base64, JSON, or Arrow IPC encoding | FR-012 KPI: <50ms serialization overhead for 10KB payload | -| **FR-013** | Transport publishing | System shall return JSON string representation for caller to publish via transport layer (caller is responsible for actual transport publish) | FR-013 KPI: 100% JSON envelope generated correctly | -| **FR-014** | Transport subscription | System shall receive and process messages by accepting JSON string from transport payload | FR-014 KPI: 100% JSON messages processed correctly | +| **FR-001** | Cross-platform text messaging | System shall allow users to send text messages between Julia, JavaScript, Python, and MicroPython applications | +| **FR-002** | Cross-platform tabular data | System shall support DataFrame exchange between Julia and Python applications using Arrow IPC format | +| **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via transport | +| **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via transport without file server upload | +| **FR-005** | MicroPython support | System shall support payloads <100KB on MicroPython devices using direct transport | +| **FR-006** | Multi-payload messages | System shall accept and process lists of (dataname, data, type) tuples | +| **FR-007** | Payload type preservation | System shall preserve payload types when returning multi-payload messages | +| **FR-008** | Plik file server integration | System shall support Plik one-shot upload mode with upload ID and token handling | +| **FR-009** | Custom file server support | System shall provide handler function abstraction for custom HTTP file server implementations | +| **FR-010** | Exponential backoff retry | System shall implement exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) for file server download failures | +| **FR-011** | Correlation ID propagation | System shall propagate correlation IDs through all message processing steps | +| **FR-012** | Message serialization | System shall serialize data types using Base64, JSON, or Arrow IPC encoding | +| **FR-013** | Transport publishing | System shall return JSON string representation for caller to publish via transport layer (caller is responsible for actual transport publish) | +| **FR-014** | Transport subscription | System shall receive and process messages by accepting JSON string from transport payload | --- @@ -458,7 +458,47 @@ function smartunpack( --- -## 15. References +## 15. Requirements Traceability Matrix + +| Requirement ID | Description | Implementation File | Test File | +|----------------|-------------|---------------------|-----------| +| FR-001 | Cross-platform text messaging | `src/msghandler.jl` | `test/test_*_sender.*` | +| FR-002 | Cross-platform tabular data | `src/msghandler.jl` | `test/test_*_sender.*` | +| FR-003 | Large file handling | `src/msghandler.jl` | `test/test_*_fileserver.*` | +| FR-004 | Direct transport for small payloads | `src/msghandler.jl` | `test/test_*_transport.*` | +| FR-005 | MicroPython support | `src/msghandler_mpy.py` | `test/test_mpy.*` | +| FR-006 | Multi-payload messages | `src/msghandler.jl` | `test/test_*_mix_payloads.*` | +| FR-007 | Payload type preservation | `src/msghandler.jl` | `test/test_*_types.*` | +| FR-008 | Plik file server integration | `src/msghandler.jl` | `test/test_plik.*` | +| FR-009 | Custom file server support | `src/msghandler.jl` | `test/test_custom_server.*` | +| FR-010 | Exponential backoff retry | `src/msghandler.jl` | `test/test_retry.*` | +| FR-011 | Correlation ID propagation | `src/msghandler.jl` | `test/test_tracing.*` | +| FR-012 | Message serialization | `src/msghandler.jl` | `test/test_*_serialization.*` | +| FR-013 | Transport publishing | `src/msghandler.jl` | `test/test_*_transport.*` | +| FR-014 | Transport subscription | `src/msghandler.jl` | `test/test_*_receiver.*` | +| NFR-101 | Message serialization overhead | `src/msghandler.jl` | `test/test_benchmarks.*` | +| NFR-102 | Message deserialization overhead | `src/msghandler.jl` | `test/test_benchmarks.*` | +| NFR-103 | Transport connection establishment | `src/msghandler.jl` | `test/test_connection.*` | +| NFR-104 | File upload latency | `src/msghandler.jl` | `test/test_fileserver.*` | +| NFR-105 | File download latency | `src/msghandler.jl` | `test/test_fileserver.*` | +| NFR-106 | Concurrent connections | `src/msghandler.jl` | `test/test_scale.*` | +| NFR-107 | Message throughput | `src/msghandler.jl` | `test/test_load.*` | +| NFR-108 | File server scalability | `docs/architecture.md` | `docs/validation.md` | +| NFR-201 | Message delivery | `src/msghandler.jl` | `test/test_delivery.*` | +| NFR-202 | File server availability | `src/msghandler.jl` | `test/test_failure_injection.*` | +| NFR-203 | Connection recovery | `src/msghandler.jl` | `test/test_reconnect.*` | +| NFR-301 | Payload integrity | `src/msghandler.jl` | `test/test_integrity.*` | +| NFR-302 | Transport security | `src/msghandler.jl` | `test/test_security.*` | +| NFR-303 | File server security | `src/msghandler.jl` | `test/test_security.*` | +| NFR-401 | Required logs | `src/msghandler.jl` | `test/test_logging.*` | +| NFR-402 | Critical metrics | `src/msghandler.jl` | `test/test_metrics.*` | +| NFR-403 | Tracing | `src/msghandler.jl` | `test/test_tracing.*` | +| NFR-404 | Alerting | `src/msghandler.jl` | `test/test_alerting.*` | +| NFR-405 | Retention | `docs/runbook.md` | `docs/validation.md` | + +--- + +## 16. References - [`src/msghandler.jl`](../src/msghandler.jl) - Ground truth implementation (Julia) - [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) - Server-side JavaScript implementation From 6171923c05ffe46342bfd851dead7ed284ad28c2 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 08:51:47 +0700 Subject: [PATCH 07/14] update --- docs/solution-design.md | 156 +++++++---- docs/specification.md | 580 +++++++++++++++++++++------------------- 2 files changed, 413 insertions(+), 323 deletions(-) diff --git a/docs/solution-design.md b/docs/solution-design.md index cc57b35..fb0ffd7 100644 --- a/docs/solution-design.md +++ b/docs/solution-design.md @@ -4,6 +4,7 @@ **Date**: 2026-05-22 **Status**: Active **Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) +**ASG Framework Alignment**: v8 pillars - Requirements → Solution Design → Specification → Walkthrough → Implementation Plan → Validation → Runbook --- @@ -11,16 +12,16 @@ msghandler addresses the challenge of cross-platform data exchange between **Julia**, **JavaScript**, **Python**, **Dart**, **Rust**, and **MicroPython** applications using message brokers as transport layers. -### Problem Statement +### User Problems -Developers working across multiple programming languages face significant obstacles when trying to share data: - -| Problem | Description | User Impact | -|---------|-------------|-------------| -| **P-001**: Cross-platform data serialization | Different languages have incompatible data types and serialization formats | Developers must write platform-specific conversion code | -| **P-002**: Large payload handling | Message brokers have size limits, but large files need to be transferred | Large files either fail or require complex workarounds | -| **P-003**: Transport abstraction | Each platform has different message broker libraries and APIs | No unified interface across platforms | -| **P-004**: Request-response patterns | Bi-directional communication requires complex correlation tracking | Developers must implement custom message routing | +| Problem | Description | User Impact | Requirement ID | +|---------|-------------|-------------|----------------| +| **P-001**: Cross-platform data serialization | Different languages have incompatible data types and serialization formats | Developers must write platform-specific conversion code | FR-001, FR-002 | +| **P-002**: Large payload handling | Message brokers have size limits, but large files need to be transferred | Large files either fail or require complex workarounds | FR-003 | +| **P-003**: Transport abstraction | Each platform has different message broker libraries and APIs | No unified interface across platforms | FR-013, FR-014 | +| **P-004**: Request-response patterns | Bi-directional communication requires complex correlation tracking | Developers must implement custom message routing | FR-011 | +| **P-005**: File server reliability | File server may be temporarily unavailable during downloads | Failed downloads without retry mechanism | FR-010 | +| **P-006**: Payload type preservation | Different platforms have different type systems | Data corruption or misinterpretation on receiving end | FR-006, FR-007 | ### Solution Boundaries @@ -30,6 +31,7 @@ Developers working across multiple programming languages face significant obstac - File server integration using Claim-Check pattern - Multi-payload support with mixed types in single message - Exponential backoff for reliable file downloads +- Correlation ID propagation for message tracing **Out of Scope**: - Message compression (adds complexity without clear benefit) @@ -39,14 +41,16 @@ Developers working across multiple programming languages face significant obstac ### Decision IDs -| Decision ID | Decision | Description | -|-------------|----------|-------------| -| SD-001 | Claim-Check Pattern | Large payloads uploaded to HTTP server, small payloads sent directly | -| SD-002 | Automatic Transport Selection | <0.5MB = direct, ≥0.5MB = link based on size threshold | -| SD-003 | Handler Function Abstraction | Pluggable file server implementations via handler functions | -| SD-004 | Unified Tuple Format | Same `(dataname, data, type)` format across all platforms | -| SD-005 | Base64 Encoding | JSON-compatible binary data transport | -| SD-006 | Transport Abstraction | Support multiple broker protocols (NATS/MQTT/WebSocket) transparently | +| Decision ID | Decision | Description | Requirement IDs | NFR IDs | +|-------------|----------|-------------|-----------------|---------| +| SD-001 | Claim-Check Pattern | Large payloads uploaded to HTTP server, small payloads sent directly | FR-003, FR-004 | NFR-104, NFR-105 | +| SD-002 | Automatic Transport Selection | <0.5MB = direct, ≥0.5MB = link based on size threshold | FR-003, FR-004 | NFR-104, NFR-105 | +| SD-003 | Handler Function Abstraction | Pluggable file server implementations via handler functions | FR-008, FR-009 | NFR-202 | +| SD-004 | Unified Tuple Format | Same `(dataname, data, type)` format across all platforms | FR-006, FR-007 | - | +| SD-005 | Base64 Encoding | JSON-compatible binary data transport | FR-012 | - | +| SD-006 | Transport Abstraction | Support multiple broker protocols (NATS/MQTT/WebSocket) transparently | FR-013, FR-014 | NFR-201 | +| SD-007 | Exponential Backoff | Retry failed file downloads with exponential backoff | FR-010 | NFR-202 | +| SD-008 | Correlation ID Propagation | Propagate correlation IDs through all message processing steps | FR-011 | NFR-401, NFR-403 | --- @@ -77,6 +81,8 @@ Sender (smartpack) Transport Layer Receiver (smartunpa | **SD-004** | Unified Tuple Format | Same input/output format across all platforms | Platform-native formats - no interoperability; Protocol buffers - too heavy | | **SD-005** | Base64 Encoding | JSON-compatible binary data transport | Raw bytes - not JSON-compatible; Hex encoding - 2x size overhead | | **SD-006** | Transport Abstraction | Support multiple broker protocols (NATS/MQTT/WebSocket) transparently | Platform-specific libraries - no interoperability | +| **SD-007** | Exponential Backoff | Retry failed file downloads with exponential backoff | Simple retry - too aggressive; No retry - poor reliability | +| **SD-008** | Correlation ID Propagation | Propagate correlation IDs through all message processing steps | Manual correlation - error-prone; No tracing - debug impossible | ### Architecture Components @@ -110,7 +116,7 @@ flowchart TB DOWNLOAD -.->|Fetch URL| API end - style CLIENT fill:#e1f5fe,stroke:#0288d1,stroke-width:2px + style Client fill:#e1f5fe,stroke:#0288d1,stroke-width:2px style Transport fill:#ffe0b2,stroke:#f57c00,stroke-width:2px style FileServer fill:#c8e6c9,stroke:#43a047,stroke-width:2px ``` @@ -190,8 +196,8 @@ flowchart TD | Component | Responsibilities | Decision IDs | Requirements Addressed | |-----------|-----------------|--------------|----------------------| | **Serialization Layer** | Convert data types to transport format (Base64/URL) | SD-005 | FR-001, FR-002, FR-012 | -| **Envelope Builder** | Create standardized message envelope with metadata | SD-001 | FR-011, FR-013, FR-014 | -| **Handler Functions** | Abstract file server operations for pluggability | SD-003 | FR-008, FR-009 | +| **Envelope Builder** | Create standardized message envelope with metadata | SD-001, SD-008 | FR-011, FR-013, FR-014 | +| **Handler Functions** | Abstract file server operations for pluggability | SD-003, SD-007 | FR-008, FR-009, FR-010 | | **Transport Adapter** | Support multiple broker protocols transparently | SD-006 | FR-013, FR-014 | | **Payload Manager** | Track payload types, sizes, and encoding | SD-004 | FR-006, FR-007 | @@ -201,7 +207,8 @@ flowchart TD ### SD-001: Why Claim-Check Pattern? -**Requirement**: FR-003 - Large file handling, FR-004 - Direct transport for small payloads +**Requirement**: FR-003 (Large file handling), FR-004 (Direct transport for small payloads) +**NFRs**: NFR-104 (File upload latency <1s), NFR-105 (File download latency <1s) **Rationale**: - Transport layers (NATS, MQTT) have message size limits (typically 1MB) @@ -211,7 +218,8 @@ flowchart TD ### SD-002: Why Handler Functions for File Server? -**Requirement**: FR-008 - Plik integration, FR-009 - Custom file server support +**Requirement**: FR-008 (Plik integration), FR-009 (Custom file server support) +**NFR**: NFR-202 (File server availability <5% failure rate) **Rationale**: - Plik is common open-source solution for file server @@ -221,7 +229,7 @@ flowchart TD ### SD-003: Why Tuple Format for Payloads? -**Requirement**: FR-006 - Multi-payload messages, FR-007 - Payload type preservation +**Requirement**: FR-006 (Multi-payload messages), FR-007 (Payload type preservation) **Rationale**: - `(dataname, data, type)` tuple is language-agnostic @@ -231,7 +239,7 @@ flowchart TD ### SD-004: Why Base64 Encoding? -**Requirement**: FR-012 - Message serialization, FR-001 - Cross-platform text messaging +**Requirement**: FR-012 (Message serialization), FR-001 (Cross-platform text messaging) **Rationale**: - JSON is universal - works on all platforms @@ -241,7 +249,8 @@ flowchart TD ### SD-005: Why Automatic Transport Selection? -**Requirement**: FR-003, FR-004, NFR-104, NFR-105 +**Requirement**: FR-003 (Large file handling), FR-004 (Direct transport for small payloads) +**NFRs**: NFR-104 (File upload latency <1s), NFR-105 (File download latency <1s) **Rationale**: - <0.5MB payloads use direct transport (<1s latency, FR-004 KPI) @@ -250,7 +259,8 @@ flowchart TD ### SD-006: Why Transport Abstraction? -**Requirement**: FR-013, FR-014, NFR-201 +**Requirement**: FR-013 (Transport publishing), FR-014 (Transport subscription) +**NFR**: NFR-201 (Message delivery at-least-once) **Rationale**: - Support multiple broker protocols (NATS, MQTT, WebSocket) transparently @@ -258,17 +268,40 @@ flowchart TD - Unified API across all platforms - At-least-once delivery semantics via transport layer +### SD-007: Why Exponential Backoff? + +**Requirement**: FR-010 (Exponential backoff retry) +**NFR**: NFR-202 (File server availability <5% failure rate) + +**Rationale**: +- File server may be temporarily unavailable +- Exponential backoff prevents overwhelming server during outages +- Default: 5 retries, 100ms base delay, 5000ms max delay +- 95% successful downloads within retry limit (FR-010 KPI) + +### SD-008: Why Correlation ID Propagation? + +**Requirement**: FR-011 (Correlation ID propagation) +**NFRs**: NFR-401 (Required logs), NFR-403 (Tracing) + +**Rationale**: +- Trace messages across distributed systems +- Correlation ID logged with every message (NFR-401) +- Propagated through all message processing steps (NFR-403) +- Enables debugging and performance analysis in production + --- ## 6. Risk Assessment -| Risk | Impact | Probability | Mitigation | -|------|--------|-------------|------------| -| **Performance degradation with >500KB payloads** | High | Medium | Size threshold detection; Link transport fallback | -| **File server availability issues** | Medium | Low | Exponential backoff retry; Graceful degradation | -| **Platform-specific bugs** | Medium | Low | Comprehensive test suite per platform; CI validation | -| **Encoding mismatches between platforms** | High | Low | Strict specification; Test contracts; Validation rules | -| **Transport layer incompatibility** | Medium | Low | Transport-agnostic design; Handler abstraction | +| Risk | Impact | Probability | Mitigation | Requirement IDs | NFR IDs | +|------|--------|-------------|------------|-----------------|---------| +| **Performance degradation with >500KB payloads** | High | Medium | Size threshold detection; Link transport fallback | FR-003, FR-004 | NFR-104, NFR-105 | +| **File server availability issues** | Medium | Low | Exponential backoff retry; Graceful degradation | FR-010 | NFR-202 | +| **Platform-specific bugs** | Medium | Low | Comprehensive test suite per platform; CI validation | FR-001, FR-002, FR-006, FR-007 | - | +| **Encoding mismatches between platforms** | High | Low | Strict specification; Test contracts; Validation rules | FR-012 | NFR-301 | +| **Transport layer incompatibility** | Medium | Low | Transport-agnostic design; Handler abstraction | FR-013, FR-014 | NFR-201 | +| **Correlation ID loss in processing** | Medium | Low | Centralized trace context management | FR-011 | NFR-401, NFR-403 | --- @@ -276,13 +309,13 @@ flowchart TD | Solution Component | Decision ID | Requirement ID | Description | |-------------------|-------------|----------------|-------------| -| **smartpack() function** | SD-001, SD-002, SD-004, SD-005, SD-006 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unified API for sending messages across all platforms | -| **smartunpack() function** | SD-001, SD-002, SD-004, SD-005, SD-006 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unified API for receiving messages across all platforms | -| **Direct transport** | SD-002 | FR-004, NFR-101, NFR-102, NFR-104, NFR-105 | Send payloads < threshold directly via transport | -| **Link transport** | SD-001, SD-002 | FR-003, NFR-104, NFR-105 | Upload payloads ≥ threshold to file server | -| **File server handler** | SD-003 | FR-008, FR-009, FR-010 | Pluggable upload/download handlers with retry logic | +| **smartpack() function** | SD-001, SD-002, SD-004, SD-005, SD-006, SD-008 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unified API for sending messages across all platforms | +| **smartunpack() function** | SD-001, SD-002, SD-004, SD-005, SD-006, SD-007, SD-008 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unified API for receiving messages across all platforms | +| **Direct transport** | SD-002 | FR-004 | Send payloads < threshold directly via transport | +| **Link transport** | SD-001, SD-002 | FR-003 | Upload payloads ≥ threshold to file server | +| **File server handler** | SD-003, SD-007 | FR-008, FR-009, FR-010 | Pluggable upload/download handlers with retry logic | | **Payload type preservation** | SD-004 | FR-006, FR-007 | Support text, dictionary, arrowtable, jsontable, image, audio, video, binary | -| **Correlation ID** | SD-001 | FR-011, NFR-401, NFR-403 | Message tracing across distributed systems | +| **Correlation ID propagation** | SD-008 | FR-011 | Message tracing across distributed systems | | **Multi-payload support** | SD-004 | FR-006, FR-007 | List of (dataname, data, type) tuples | ### Non-Functional Requirements Traceability @@ -295,11 +328,15 @@ flowchart TD | **Concurrent connections** | SD-006 | NFR-106 | Support 100+ simultaneous connections | | **Message throughput** | SD-005, SD-006 | NFR-107 | Handle 1000+ messages/second per instance | | **At-least-once delivery** | SD-006 | NFR-201 | Transport layer semantics | -| **Graceful degradation** | SD-003 | NFR-202 | File server unavailability handling | +| **Graceful degradation** | SD-003, SD-007 | NFR-202 | File server unavailability handling | | **Auto-reconnect** | SD-006 | NFR-203 | Transport connection failure recovery | -| **Required logs** | SD-001 | NFR-401 | Correlation ID, msg_id, timestamp, etc. | +| **Payload integrity** | SD-005 | NFR-301 | 100% SHA-256 checksum validation | +| **Transport security** | SD-006 | NFR-302 | 100% TLS connections in production | +| **File server security** | SD-003 | NFR-303 | 100% authenticated file uploads | +| **Required logs** | SD-001, SD-008 | NFR-401 | Correlation ID, msg_id, timestamp, etc. | | **Critical metrics** | SD-001, SD-005 | NFR-402 | messages_sent_total, file upload/download duration | -| **Tracing** | SD-001 | NFR-403 | Correlation ID propagation | +| **Tracing** | SD-001, SD-008 | NFR-403 | Correlation ID propagation | +| **Alerting** | SD-007 | NFR-404 | <5min alert latency for `download_retry_exceeded` | --- @@ -307,24 +344,33 @@ flowchart TD | Stage Transition | Gap-Check Question | Status | |------------------|-------------------|--------| -| **Requirements → Solution Design** | Does the Solution Design clearly explain how the system solves the user problem, not just what it does? | ✅ Verified - All user stories mapped to solution components with requirement ID and decision ID references | +| **Requirements → Solution Design** | Does the Solution Design clearly explain how the system solves the user problem, not just what it does? | ✅ Verified - All user problems mapped to solution components with requirement ID and decision ID references | | **Solution Design → Specification** | Does the Specification define all technical details that the solution approach requires? | ⏳ Pending - Specification needs review for completeness | | **Solution Design → Walkthrough** | Does the Walkthrough reflect the complete flow including error states and timing? | ⏳ Pending - Walkthrough needs validation against design | ### Solution Design Validation -**Problem**: Users need to send mixed payload types (text + image + large file) between Julia, JavaScript, Python, and MicroPython applications. +**User Problems** (from requirements.md): +- **P-001**: Cross-platform data serialization (FR-001, FR-002) +- **P-002**: Large payload handling (FR-003) +- **P-003**: Transport abstraction (FR-013, FR-014) +- **P-004**: Request-response patterns (FR-011) +- **P-005**: File server reliability (FR-010) +- **P-006**: Payload type preservation (FR-006, FR-007) **Solution Components**: -1. **SD-001** - `smartpack()` - Unified API for all platforms -2. **SD-002** - Tuple format - `(dataname, data, type)` - platform-agnostic -3. **SD-003** - Automatic transport selection - <0.5MB = direct, ≥0.5MB = link -4. **SD-004** - File server handler abstraction - Plik/AWS S3/custom support -5. **SD-005** - Exponential backoff - Reliable file downloads -6. **SD-006** - Correlation ID - Message tracing +1. **SD-001** - `smartpack()` / `smartunpack()` - Unified API for all platforms +2. **SD-002** - Claim-Check pattern - Automatic transport selection based on size threshold +3. **SD-003** - Handler function abstraction - Plik/AWS S3/custom file server support +4. **SD-004** - Tuple format - `(dataname, data, type)` - platform-agnostic +5. **SD-005** - Base64 encoding - JSON-compatible binary data transport +6. **SD-006** - Transport abstraction - Support multiple broker protocols transparently +7. **SD-007** - Exponential backoff - Reliable file downloads with retry logic +8. **SD-008** - Correlation ID propagation - Message tracing across distributed systems **Requirement Mapping**: -- FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 ✅ +- **Functional Requirements**: FR-001 through FR-014 ✅ +- **Non-Functional Requirements**: NFR-101 through NFR-405 ✅ **Gap Check**: Does this solution explain *how* users will actually use the system? @@ -333,7 +379,13 @@ flowchart TD 2. `smartpack()` automatically selects transport based on size (SD-002) 3. Large file (≥0.5MB) → link transport → file server upload (SD-001) 4. Small payload (<0.5MB) → direct transport → base64 encoding (SD-005) -5. Receiver calls `smartunpack()` → receives same tuple format +5. Receiver calls `smartunpack()` → receives same tuple format with preserved types + +**NFR Traceability**: +- **Performance**: NFR-101 (serialization <50ms), NFR-102 (deserialization <50ms), NFR-103 (connection <100ms) ✅ +- **Reliability**: NFR-201 (at-least-once delivery), NFR-202 (file server <5% failure), NFR-203 (auto-reconnect <30s) ✅ +- **Security**: NFR-301 (SHA-256 checksum), NFR-302 (TLS 100%), NFR-303 (authenticated uploads) ✅ +- **Observability**: NFR-401 (required logs), NFR-402 (metrics), NFR-403 (tracing), NFR-404 (alerting <5min) ✅ --- diff --git a/docs/specification.md b/docs/specification.md index df773f6..101e698 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -1,10 +1,11 @@ # Specification: msghandler -**Version**: 1.2.0 -**Date**: 2026-05-13 +**Version**: 1.3.0 +**Date**: 2026-05-22 **Status**: Active **Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) -**Specification Format**: JSON Schema + AsyncAPI +**Specification Format**: JSON Schema + AsyncAPI +**ASG Framework Alignment**: v8 pillars - Requirements → Solution Design → Specification → Walkthrough → Implementation Plan → Validation → Runbook --- @@ -20,26 +21,26 @@ This specification serves as the single source of truth for: ### 1.1 Requirements Traceability -| Specification Section | Requirement ID(s) | Description | -|----------------------|-------------------|-------------| -| Section 2 (Message Envelope) | FR-012, FR-013, NFR-101, NFR-102 | Message envelope structure and validation | -| Section 3 (Payload Schema) | FR-001, FR-002, FR-003, FR-004, NFR-101, NFR-102 | Payload structure and field definitions | -| Section 4 (Payload Format) | FR-006, FR-007 | Tuple format for smartpack() | -| Section 5 (Enumerations) | FR-003, FR-004, FR-006, NFR-101 | Enumerations for transport and encoding | -| Section 6 (Transport Protocols) | FR-003, FR-004, NFR-104, NFR-105 | Direct and link transport protocols | -| Section 7 (Size Thresholds) | FR-004, FR-005, NFR-104, NFR-105 | Size thresholds for transport selection | -| Section 8 (Topic Convention) | FR-013, FR-014 | Topic/subject naming patterns | -| Section 9 (Error Handling) | FR-010, FR-011, NFR-201, NFR-202, NFR-203 | Error codes and exception handling | -| Section 10 (Serialization Rules) | FR-001, FR-002, FR-003, FR-012, NFR-101, NFR-102 | Serialization and encoding rules | -| Section 11 (API Contract) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Function signatures for all platforms | -| Section 12 (File Server Interface) | FR-008, FR-009, FR-010 | Upload and download handler contracts | -| Section 13 (Platform-Specific Constraints) | FR-005, FR-006, NFR-106, NFR-107 | Platform-specific feature support | -| Section 14 (Implementation Files) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007 | Implementation file mapping | -| Section 15 (Message Flow) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Mermaid diagrams for send/receive flows | -| Section 16 (Validation Rules) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Envelope and payload validation rules | -| Section 17 (Test Contracts) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unit and integration test scenarios | -| Section 18 (Dependencies) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Platform-specific dependencies | -| Section 19 (Change Log) | N/A | Version history and changes | +| Specification Section | Requirement ID(s) | Solution Design Ref(s) | Description | +|----------------------|-------------------|------------------------|-------------| +| Section 2 (Message Envelope) | FR-012, FR-013, NFR-101, NFR-102 | SD-008 | Message envelope structure and validation | +| Section 3 (Payload Schema) | FR-001, FR-002, FR-003, FR-004, NFR-101, NFR-102 | SD-001, SD-005 | Payload structure and field definitions | +| Section 4 (Payload Format) | FR-006, FR-007 | SD-004 | Tuple format for smartpack() | +| Section 5 (Enumerations) | FR-003, FR-004, FR-006, NFR-101 | SD-001, SD-002, SD-005 | Enumerations for transport and encoding | +| Section 6 (Transport Protocols) | FR-003, FR-004, NFR-104, NFR-105 | SD-001, SD-002 | Direct and link transport protocols | +| Section 7 (Size Thresholds) | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | Size thresholds for transport selection | +| Section 8 (Topic Convention) | FR-013, FR-014 | SD-006 | Topic/subject naming patterns | +| Section 9 (Error Handling) | FR-010, FR-011, NFR-201, NFR-202, NFR-203 | SD-003, SD-007 | Error codes and exception handling | +| Section 10 (Serialization Rules) | FR-001, FR-002, FR-003, FR-012, NFR-101, NFR-102 | SD-005 | Serialization and encoding rules | +| Section 11 (API Contract) | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | Function signatures for all platforms | +| Section 12 (File Server Interface) | FR-008, FR-009, FR-010 | SD-003, SD-007 | Upload and download handler contracts | +| Section 13 (Platform-Specific Constraints) | FR-005, FR-006, NFR-106, NFR-107 | SD-004, SD-006 | Platform-specific feature support | +| Section 14 (Implementation Files) | FR-001 through FR-007 | - | Implementation file mapping | +| Section 15 (Message Flow) | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | Mermaid diagrams for send/receive flows | +| Section 16 (Validation Rules) | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | Envelope and payload validation rules | +| Section 17 (Test Contracts) | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | Unit and integration test scenarios | +| Section 18 (Dependencies) | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | Platform-specific dependencies | +| Section 19 (Change Log) | - | - | Version history and changes | --- @@ -90,22 +91,22 @@ This specification serves as the single source of truth for: ### Field Definitions -| Field | Type | Required | Validation | Description | Requirement ID | -|-------|------|----------|------------|-------------|----------------| -| `correlation_id` | `string` | Yes | UUID v4 format | Track message flow across distributed systems | FR-011, NFR-401 | -| `msg_id` | `string` | Yes | UUID v4 format | Unique identifier for this specific message | FR-012, NFR-401 | -| `timestamp` | `string` | Yes | ISO 8601 UTC | Message publication timestamp | FR-012, NFR-401 | -| `send_to` | `string` | Yes | Non-empty string | Topic/subject to publish the message to | FR-013 | -| `msg_purpose` | `string` | Yes | Enum | Purpose of the message | FR-013 | -| `sender_name` | `string` | Yes | Non-empty string | Name of the sender application | FR-013 | -| `sender_id` | `string` | Yes | UUID v4 format | Unique identifier for the sender | FR-013 | -| `receiver_name` | `string` | Yes | Any string | Name of the receiver (empty = broadcast) | FR-013 | -| `receiver_id` | `string` | Yes | Any string | UUID of the receiver (empty = broadcast) | FR-013 | -| `reply_to` | `string` | Yes | Any string | Topic where receiver should reply | FR-013 | -| `reply_to_msg_id` | `string` | Yes | Any string | Message ID this message is replying to | FR-013 | -| `broker_url` | `string` | Yes | Valid URL | Broker URL for the transport layer | FR-013 | -| `metadata` | `object` | No | Any JSON object | Message-level metadata | NFR-401 | -| `payloads` | `array` | Yes | Non-empty array | List of payload objects | FR-012, FR-013 | +| Field | Type | Required | Validation | Description | Requirement ID | Solution Design Ref | +|-------|------|----------|------------|-------------|----------------|-------------------| +| `correlation_id` | `string` | Yes | UUID v4 format | Track message flow across distributed systems | FR-011, NFR-401 | SD-008 | +| `msg_id` | `string` | Yes | UUID v4 format | Unique identifier for this specific message | FR-012, NFR-401 | SD-001 | +| `timestamp` | `string` | Yes | ISO 8601 UTC | Message publication timestamp | FR-012, NFR-401 | SD-001 | +| `send_to` | `string` | Yes | Non-empty string | Topic/subject to publish the message to | FR-013 | SD-006 | +| `msg_purpose` | `string` | Yes | Enum | Purpose of the message | FR-013 | SD-006 | +| `sender_name` | `string` | Yes | Non-empty string | Name of the sender application | FR-013 | SD-006 | +| `sender_id` | `string` | Yes | UUID v4 format | Unique identifier for the sender | FR-013 | SD-006 | +| `receiver_name` | `string` | Yes | Any string | Name of the receiver (empty = broadcast) | FR-013 | SD-006 | +| `receiver_id` | `string` | Yes | Any string | UUID of the receiver (empty = broadcast) | FR-013 | SD-006 | +| `reply_to` | `string` | Yes | Any string | Topic where receiver should reply | FR-013 | SD-006 | +| `reply_to_msg_id` | `string` | Yes | Any string | Message ID this message is replying to | FR-013 | SD-006 | +| `broker_url` | `string` | Yes | Valid URL | Broker URL for the transport layer | FR-013 | SD-006 | +| `metadata` | `object` | No | Any JSON object | Message-level metadata | NFR-401 | SD-001, SD-008 | +| `payloads` | `array` | Yes | Non-empty array | List of payload objects | FR-012, FR-013 | SD-001, SD-005 | --- @@ -128,16 +129,16 @@ This specification serves as the single source of truth for: ### Payload Field Definitions -| Field | Type | Required | Validation | Description | Requirement ID | -|-------|------|----------|------------|-------------|----------------| -| `id` | `string` | Yes | UUID v4 format | Unique identifier for this payload | FR-012 | -| `dataname` | `string` | Yes | Non-empty string | Name of the payload (e.g., `login_image`, `user_data`) | FR-001, FR-002, FR-003 | -| `payload_type` | `string` | Yes | Enum | Type of payload (see `payload_type` enum) | FR-001, FR-002, FR-003, FR-006 | -| `transport` | `string` | Yes | Enum | Transport method: `direct` or `link` | FR-003, FR-004, NFR-104, NFR-105 | -| `encoding` | `string` | Yes | Enum | Encoding method (see `encoding` enum) | FR-001, FR-002, FR-003, FR-012 | -| `size` | `integer` | Yes | Positive integer | Size of the payload in bytes | FR-003, FR-004, NFR-104, NFR-105 | -| `data` | `string` or `URL` | Yes | Base64 string or URL | Payload data (base64 for direct, URL for link) | FR-003, FR-004, FR-008, FR-009, FR-010 | -| `metadata` | `object` | No | Any JSON object | Payload-level metadata | NFR-401 | +| Field | Type | Required | Validation | Description | Requirement ID | Solution Design Ref | +|-------|------|----------|------------|-------------|----------------|-------------------| +| `id` | `string` | Yes | UUID v4 format | Unique identifier for this payload | FR-012 | SD-001 | +| `dataname` | `string` | Yes | Non-empty string | Name of the payload (e.g., `login_image`, `user_data`) | FR-001, FR-002, FR-003 | SD-004 | +| `payload_type` | `string` | Yes | Enum | Type of payload (see `payload_type` enum) | FR-001, FR-002, FR-003, FR-006 | SD-004, SD-005 | +| `transport` | `string` | Yes | Enum | Transport method: `direct` or `link` | FR-003, FR-004, NFR-104, NFR-105 | SD-001, SD-002 | +| `encoding` | `string` | Yes | Enum | Encoding method (see `encoding` enum) | FR-001, FR-002, FR-003, FR-012 | SD-005 | +| `size` | `integer` | Yes | Positive integer | Size of the payload in bytes | FR-003, FR-004, NFR-104, NFR-105 | SD-002 | +| `data` | `string` or `URL` | Yes | Base64 string or URL | Payload data (base64 for direct, URL for link) | FR-003, FR-004, FR-008, FR-009, FR-010 | SD-001, SD-003 | +| `metadata` | `object` | No | Any JSON object | Payload-level metadata | NFR-401 | SD-008 | --- @@ -227,30 +228,30 @@ await smartpack("/agent/v1/process", data) | Value | Description | Supported Platforms | Encoding Options | |-------|-------------|---------------------|------------------| -| `text` | Plain text string | All | `base64` | -| `dictionary` | JSON object/dictionary | All | `base64`, `json` | -| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/Python/Node.js/Dart) | `base64`, `arrow-ipc` | -| `jsontable` | JSON array of objects | All (including Browser/Dart Web) | `base64`, `json` | -| `image` | Binary image data | All | `base64` | -| `audio` | Binary audio data | All | `base64` | -| `video` | Binary video data | All | `base64` | -| `binary` | Generic binary data | All | `base64` | +| `text` | Plain text string | All | `base64` | FR-001, FR-012 | SD-004, SD-005 | +| `dictionary` | JSON object/dictionary | All | `base64`, `json` | FR-002, FR-012 | SD-004, SD-005 | +| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/Python/Node.js/Dart) | `base64`, `arrow-ipc` | FR-002 | SD-004, SD-005 | +| `jsontable` | JSON array of objects | All (including Browser/Dart Web) | `base64`, `json` | FR-002, FR-006 | SD-004, SD-005 | +| `image` | Binary image data | All | `base64` | FR-001, FR-006 | SD-004, SD-005 | +| `audio` | Binary audio data | All | `base64` | FR-001, FR-006 | SD-004, SD-005 | +| `video` | Binary video data | All | `base64` | FR-001, FR-006 | SD-004, SD-005 | +| `binary` | Generic binary data | All | `base64` | FR-001, FR-006 | SD-004, SD-005 | ### `transport` Enum | Value | Description | Data Format | Use Case | |-------|-------------|-------------|----------| -| `direct` | Payload sent directly via the transport layer | Base64-encoded string | Payloads < size_threshold | -| `link` | Payload uploaded to file server | HTTP URL | Payloads ≥ size_threshold | +| `direct` | Payload sent directly via the transport layer | Base64-encoded string | Payloads < size_threshold | FR-003, FR-004, NFR-104, NFR-105 | SD-001, SD-002 | +| `link` | Payload uploaded to file server | HTTP URL | Payloads ≥ size_threshold | FR-003, FR-004, NFR-104, NFR-105 | SD-001, SD-002 | ### `encoding` Enum | Value | Description | Payload Types | |-------|-------------|---------------| -| `none` | No additional encoding | Link transport URLs | -| `base64` | Base64 encoding | Text, binary, image, audio, video | -| `json` | JSON encoding | Dictionary, jsontable | -| `arrow-ipc` | Apache Arrow IPC format | Arrowtable | +| `none` | No additional encoding | Link transport URLs | FR-003, FR-004 | SD-001, SD-002 | +| `base64` | Base64 encoding | Text, binary, image, audio, video | FR-001, FR-002, FR-003, FR-012 | SD-005 | +| `json` | JSON encoding | Dictionary, jsontable | FR-002, FR-006 | SD-004, SD-005 | +| `arrow-ipc` | Apache Arrow IPC format | Arrowtable | FR-002 | SD-004, SD-005 | --- @@ -300,15 +301,15 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa ### Desktop Platforms (Julia/JS/Python) -| Platform | Size Threshold | Notes | -|----------|----------------|-------| -| Desktop | 500,000 bytes (0.5MB) | Default threshold | +| Platform | Size Threshold | Notes | Requirement ID | Solution Design Ref | +|----------|----------------|-------|----------------|-------------------| +| Desktop | 500,000 bytes (0.5MB) | Default threshold | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | ### MicroPython Platform -| Platform | Size Threshold | Maximum Payload | Notes | -|----------|----------------|-----------------|-------| -| MicroPython | 100,000 bytes (100KB) | 50,000 bytes | Hard limit due to memory constraints | +| Platform | Size Threshold | Maximum Payload | Notes | Requirement ID | Solution Design Ref | +|----------|----------------|-----------------|-------|----------------|-------------------| +| MicroPython | 100,000 bytes (100KB) | 50,000 bytes | Hard limit due to memory constraints | FR-005, NFR-106 | SD-002 | --- @@ -321,16 +322,16 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa ``` **Examples**: -- `/agent/wine/api/v1/prompt` - AI agent prompt endpoint -- `/chat/user/v1/message` - User chat message -- `/system/worker/v1/status` - Worker status update +- `/agent/wine/api/v1/prompt` - AI agent prompt endpoint | FR-013 | SD-006 +- `/chat/user/v1/message` - User chat message | FR-013 | SD-006 +- `/system/worker/v1/status` - Worker status update | FR-013 | SD-006 ### Subject Wildcards | Wildcard | Description | Example | |----------|-------------|---------| -| `*` | Single-level wildcard | `/chat/user/v1/*` matches `/chat/user/v1/message` | -| `>` | Multi-level wildcard | `/chat/user/v1/>` matches all `/chat/user/v1/*` subjects | +| `*` | Single-level wildcard | `/chat/user/v1/*` matches `/chat/user/v1/message` | FR-013 | SD-006 | +| `>` | Multi-level wildcard | `/chat/user/v1/>` matches all `/chat/user/v1/*` subjects | FR-013 | SD-006 | --- @@ -352,23 +353,23 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa | Code | HTTP Status | Description | Recovery | Requirement ID | |------|-------------|-------------|----------|----------------| -| `INVALID_ENVELOPE` | 400 | Message envelope validation failed | Fix envelope structure | FR-012, FR-013, FR-014 | -| `INVALID_PAYLOAD_TYPE` | 400 | Unsupported payload type | Use supported payload_type | FR-001, FR-002, FR-003, FR-006 | -| `INVALID_TRANSPORT` | 400 | Unsupported transport type | Use `direct` or `link` | FR-003, FR-004, FR-006 | -| `UPLOAD_FAILED` | 500 | File server upload failed | Retry or use direct transport | FR-008, FR-009 | -| `DOWNLOAD_FAILED` | 503 | File server download failed | Retry with exponential backoff | FR-010, FR-011, NFR-201, NFR-202 | -| `TRANSPORT_CONNECTION_FAILED` | 503 | Transport connection failed | Check broker/server availability | FR-013, FR-014, NFR-201, NFR-203 | -| `DESERIALIZATION_ERROR` | 500 | Payload deserialization failed | Check payload_type matches data | FR-001, FR-002, FR-003, FR-012 | -| `SIZE_EXCEEDED` | 413 | Payload exceeds maximum size | Split payload or use link transport | FR-003, FR-004, FR-005, NFR-104, NFR-105 | +| `INVALID_ENVELOPE` | 400 | Message envelope validation failed | Fix envelope structure | FR-012, FR-013, FR-014 | SD-001, SD-006 | +| `INVALID_PAYLOAD_TYPE` | 400 | Unsupported payload type | Use supported payload_type | FR-001, FR-002, FR-003, FR-006 | SD-004, SD-005 | +| `INVALID_TRANSPORT` | 400 | Unsupported transport type | Use `direct` or `link` | FR-003, FR-004, FR-006 | SD-001, SD-002 | +| `UPLOAD_FAILED` | 500 | File server upload failed | Retry or use direct transport | FR-008, FR-009 | SD-003 | +| `DOWNLOAD_FAILED` | 503 | File server download failed | Retry with exponential backoff | FR-010, FR-011, NFR-201, NFR-202 | SD-003, SD-007 | +| `TRANSPORT_CONNECTION_FAILED` | 503 | Transport connection failed | Check broker/server availability | FR-013, FR-014, NFR-201, NFR-203 | SD-006 | +| `DESERIALIZATION_ERROR` | 500 | Payload deserialization failed | Check payload_type matches data | FR-001, FR-002, FR-003, FR-012 | SD-004, SD-005 | +| `SIZE_EXCEEDED` | 413 | Payload exceeds maximum size | Split payload or use link transport | FR-003, FR-004, FR-005, NFR-104, NFR-105 | SD-001, SD-002 | ### Exception Handling | Scenario | Handler | Retry Policy | Requirement ID | |----------|---------|--------------|----------------| -| File server unavailable | Retry up to 5 times | Exponential backoff (100ms → 5000ms) | FR-010, NFR-202 | -| Transport publish failure | Handled by caller | Caller's retry logic | FR-013, FR-014, NFR-201, NFR-203 | -| Deserialization error | Log correlation ID and throw | No retry (data corruption) | FR-001, FR-002, FR-003, FR-012, NFR-401 | -| Memory overflow (MicroPython) | Reject payloads >50KB | No retry (client-side check) | FR-005, NFR-106 | +| File server unavailable | Retry up to 5 times | Exponential backoff (100ms → 5000ms) | FR-010, NFR-202 | SD-003, SD-007 | +| Transport publish failure | Handled by caller | Caller's retry logic | FR-013, FR-014, NFR-201, NFR-203 | SD-006 | +| Deserialization error | Log correlation ID and throw | No retry (data corruption) | FR-001, FR-002, FR-003, FR-012, NFR-401 | SD-004, SD-005, SD-008 | +| Memory overflow (MicroPython) | Reject payloads >50KB | No retry (client-side check) | FR-005, NFR-106 | SD-004 | --- @@ -411,6 +412,11 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa ## API Contract +| Platform | Function | Requirements | Solution Design Ref | +|----------|----------|--------------|-------------------| +| All | `smartpack()` | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| All | `smartunpack()` | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | + ### `smartpack` Function Signature #### Julia @@ -638,6 +644,10 @@ pub struct MsgEnvelopeV1 { ### `smartunpack` Function Signature +**Note**: Input is the JSON string payload from the transport subscription. Returns `(envelope::msg_envelope_v1, payloads::Array{Tuple{String, Any, String}, 1})`. + +**Traceability**: FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 + #### Julia ```julia @@ -757,6 +767,8 @@ pub struct smartunpackOptions { ## File Server Interface +**Traceability**: FR-008, FR-009, FR-010, NFR-202 | SD-003, SD-007 + ### Upload Handler Contract **Function Signature**: @@ -785,15 +797,19 @@ function fileserver_upload_handler( ``` **Required Keys**: -| Key | Type | Description | -|-----|------|-------------| -| `status` | `integer` | HTTP response status code | -| `uploadid` | `string` | Upload session identifier | -| `fileid` | `string` | File identifier within session | -| `url` | `string` | Full download URL | +| Key | Type | Description | Requirement ID | Solution Design Ref | +|-----|------|-------------|----------------|-------------------| +| `status` | `integer` | HTTP response status code | FR-008, FR-009 | SD-003 | +| `uploadid` | `string` | Upload session identifier | FR-008 | SD-003 | +| `fileid` | `string` | File identifier within session | FR-008 | SD-003 | +| `url` | `string` | Full download URL | FR-008, FR-009, FR-010 | SD-003, SD-007 | ### Download Handler Contract +**Function Signature**: + +**Traceability**: FR-010, NFR-202 | SD-003, SD-007 + **Function Signature**: ```julia function fileserver_download_handler( @@ -815,102 +831,108 @@ function fileserver_download_handler( ## Platform-Specific Constraints +**Traceability**: FR-005, FR-006, NFR-106, NFR-107 | SD-002, SD-004 + ### Desktop (Julia/Python/Node.js/Dart) -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ✅ Supported | Requires Arrow.jl/pyarrow/dart-arrow | -| JSON table | ✅ Supported | Human-readable format | -| File server upload | ✅ Supported | HTTP/HTTPS | -| File server download | ✅ Supported | HTTP/HTTPS | -| Size threshold | 500KB | Configurable | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ✅ Supported | Requires Arrow.jl/pyarrow/dart-arrow | FR-002, FR-012 | SD-004, SD-005 | +| JSON table | ✅ Supported | Human-readable format | FR-002, FR-006 | SD-004, SD-005 | +| File server upload | ✅ Supported | HTTP/HTTPS | FR-008, FR-009 | SD-003 | +| File server download | ✅ Supported | HTTP/HTTPS | FR-010, NFR-202 | SD-003, SD-007 | +| Size threshold | 500KB | Configurable | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | ### Browser (JavaScript) -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible | -| JSON table | ✅ Supported | Only table type available in browser | -| File server upload | ✅ Supported | HTTP/HTTPS | -| File server download | ✅ Supported | HTTP/HTTPS | -| Size threshold | 500KB | Configurable | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible | FR-002 | SD-004, SD-005 | +| JSON table | ✅ Supported | Only table type available in browser | FR-002, FR-006 | SD-004, SD-005 | +| File server upload | ✅ Supported | HTTP/HTTPS | FR-008, FR-009 | SD-003 | +| File server download | ✅ Supported | HTTP/HTTPS | FR-010, NFR-202 | SD-003, SD-007 | +| Size threshold | 500KB | Configurable | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | ### Dart Desktop (Dart SDK) -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ✅ Supported | Requires dart-arrow package | -| JSON table | ✅ Supported | Human-readable format | -| File server upload | ✅ Supported | HTTP/HTTPS | -| File server download | ✅ Supported | HTTP/HTTPS | -| Size threshold | 500KB | Configurable | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ✅ Supported | Requires dart-arrow package | FR-002, FR-012 | SD-004, SD-005 | +| JSON table | ✅ Supported | Human-readable format | FR-002, FR-006 | SD-004, SD-005 | +| File server upload | ✅ Supported | HTTP/HTTPS | FR-008, FR-009 | SD-003 | +| File server download | ✅ Supported | HTTP/HTTPS | FR-010, NFR-202 | SD-003, SD-007 | +| Size threshold | 500KB | Configurable | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | ### Dart Flutter (Dart SDK) -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ✅ Supported | Requires dart-arrow package | -| JSON table | ✅ Supported | Human-readable format | -| File server upload | ✅ Supported | HTTP/HTTPS | -| File server download | ✅ Supported | HTTP/HTTPS | -| Size threshold | 500KB | Configurable | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ✅ Supported | Requires dart-arrow package | FR-002, FR-012 | SD-004, SD-005 | +| JSON table | ✅ Supported | Human-readable format | FR-002, FR-006 | SD-004, SD-005 | +| File server upload | ✅ Supported | HTTP/HTTPS | FR-008, FR-009 | SD-003 | +| File server download | ✅ Supported | HTTP/HTTPS | FR-010, NFR-202 | SD-003, SD-007 | +| Size threshold | 500KB | Configurable | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | ### Dart Web (Dart SDK) -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible | -| JSON table | ✅ Supported | Only table type available in browser | -| File server upload | ✅ Supported | HTTP/HTTPS | -| File server download | ✅ Supported | HTTP/HTTPS | -| Size threshold | 500KB | Configurable | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible | FR-002 | SD-004, SD-005 | +| JSON table | ✅ Supported | Only table type available in browser | FR-002, FR-006 | SD-004, SD-005 | +| File server upload | ✅ Supported | HTTP/HTTPS | FR-008, FR-009 | SD-003 | +| File server download | ✅ Supported | HTTP/HTTPS | FR-010, NFR-202 | SD-003, SD-007 | +| Size threshold | 500KB | Configurable | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | ### Rust -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ✅ Supported | Requires `arrow2` crate | -| JSON table | ✅ Supported | Uses `serde_json` | -| File server upload | ✅ Supported | HTTP/HTTPS via `reqwest` | -| File server download | ✅ Supported | HTTP/HTTPS via `reqwest` with retry | -| Size threshold | 500KB | Configurable | -| Async runtime | ✅ Supported | Uses `tokio` for async I/O | -| Type safety | ✅ Supported | Compile-time type checking via Rust enums | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ✅ Supported | Requires `arrow2` crate | FR-002, FR-012 | SD-004, SD-005 | +| JSON table | ✅ Supported | Uses `serde_json` | FR-002, FR-006 | SD-004, SD-005 | +| File server upload | ✅ Supported | HTTP/HTTPS via `reqwest` | FR-008, FR-009 | SD-003 | +| File server download | ✅ Supported | HTTP/HTTPS via `reqwest` with retry | FR-010, NFR-202 | SD-003, SD-007 | +| Size threshold | 500KB | Configurable | FR-004, FR-005, NFR-104, NFR-105 | SD-002 | +| Async runtime | ✅ Supported | Uses `tokio` for async I/O | FR-013, FR-014 | SD-006 | +| Type safety | ✅ Supported | Compile-time type checking via Rust enums | FR-006, FR-007 | SD-004 | ### MicroPython -| Feature | Status | Notes | -|---------|--------|-------| -| Arrow IPC | ❌ Not supported | Memory constraints | -| JSON table | ⚠️ Limited | Only direct transport | -| File server upload | ❌ Not implemented | Placeholder only | -| File server download | ❌ Not implemented | Placeholder only | -| Size threshold | 100KB | Hard limit enforced | -| Max payload | 50KB | Hard limit enforced | +| Feature | Status | Notes | Requirement ID | Solution Design Ref | +|---------|--------|-------|----------------|-------------------| +| Arrow IPC | ❌ Not supported | Memory constraints | FR-005 | SD-002 | +| JSON table | ⚠️ Limited | Only direct transport | FR-005, FR-006 | SD-002, SD-004 | +| File server upload | ❌ Not implemented | Placeholder only | FR-005 | SD-002 | +| File server download | ❌ Not implemented | Placeholder only | FR-005 | SD-002 | +| Size threshold | 100KB | Hard limit enforced | FR-005, NFR-106 | SD-002 | +| Max payload | 50KB | Hard limit enforced | FR-005 | SD-002 | --- ## Implementation Files -| File | Platform | Features | Notes | -|------|----------|----------|-------| -| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | Ground truth implementation | -| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | Server-side JavaScript | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | Client-side rendering | -| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | Desktop Python | -| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | Desktop/Flutter/Web | -| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe | Uses tokio + serde + arrow2 | -| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | Memory-constrained | +**Traceability**: FR-001 through FR-007 | SD-001 through SD-008 + +| File | Platform | Features | Notes | Requirement ID | Solution Design Ref | +|------|----------|----------|-------|----------------|-------------------| +| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | Ground truth implementation | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | Server-side JavaScript | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | Client-side rendering | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | Desktop Python | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | Desktop/Flutter/Web | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe | Uses tokio + serde + arrow2 | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | Memory-constrained | FR-005, FR-006 | SD-002, SD-004 | ### Browser Implementation Notes +**Traceability**: FR-002, FR-006 | SD-004, SD-005 + The browser implementation ([`src/msghandler_csr.js`](../src/msghandler_csr.js)) has the following constraints: -| Constraint | Reason | Workaround | -|------------|--------|------------| -| No Apache Arrow IPC | Browser-incompatible dependency | Use `jsontable` for tabular data | -| WebSocket only | Browser cannot use TCP directly | Use `ws://` or `wss://` broker URLs | -| Fetch API for HTTP | Browser fetch() API only | Compatible with Plik and other HTTP servers | +| Constraint | Reason | Workaround | Requirement ID | Solution Design Ref | +|------------|--------|------------|----------------|-------------------| +| No Apache Arrow IPC | Browser-incompatible dependency | Use `jsontable` for tabular data | FR-002 | SD-004, SD-005 | +| WebSocket only | Browser cannot use TCP directly | Use `ws://` or `wss://` broker URLs | FR-013, FR-014 | SD-006 | +| Fetch API for HTTP | Browser fetch() API only | Compatible with Plik and other HTTP servers | FR-008, FR-009 | SD-003 | ### Payload Type Availability by Platform @@ -975,40 +997,44 @@ flowchart TD ## Validation Rules +**Traceability**: FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 + ### Envelope Validation -| Rule | Condition | Error Code | Requirement ID | -|------|-----------|------------|----------------| -| Required fields present | `correlation_id`, `msg_id`, `timestamp`, `send_to`, `payloads` | `INVALID_ENVELOPE` | FR-012, FR-013 | -| Valid UUID format | `correlation_id`, `msg_id`, `sender_id`, `receiver_id` | `INVALID_ENVELOPE` | FR-011, FR-012, NFR-401 | -| Valid timestamp format | ISO 8601 UTC | `INVALID_ENVELOPE` | FR-012, NFR-401 | -| Non-empty payloads array | `length(payloads) > 0` | `INVALID_ENVELOPE` | FR-012, FR-013 | +| Rule | Condition | Error Code | Requirement ID | Solution Design Ref | +|------|-----------|------------|----------------|-------------------| +| Required fields present | `correlation_id`, `msg_id`, `timestamp`, `send_to`, `payloads` | `INVALID_ENVELOPE` | FR-012, FR-013 | SD-001, SD-006 | +| Valid UUID format | `correlation_id`, `msg_id`, `sender_id`, `receiver_id` | `INVALID_ENVELOPE` | FR-011, FR-012, NFR-401 | SD-001, SD-008 | +| Valid timestamp format | ISO 8601 UTC | `INVALID_ENVELOPE` | FR-012, NFR-401 | SD-001, SD-008 | +| Non-empty payloads array | `length(payloads) > 0` | `INVALID_ENVELOPE` | FR-012, FR-013 | SD-001, SD-005 | ### Payload Validation -| Rule | Condition | Error Code | Requirement ID | -|------|-----------|------------|----------------| -| Valid payload_type | Must be in `payload_type` enum | `INVALID_PAYLOAD_TYPE` | FR-001, FR-002, FR-003, FR-006 | -| Valid transport | Must be `direct` or `link` | `INVALID_TRANSPORT` | FR-003, FR-004, FR-006 | -| Valid encoding | Must match payload_type and transport | `INVALID_TRANSPORT` | FR-001, FR-002, FR-003, FR-012 | -| Positive size | `size > 0` | `INVALID_PAYLOAD` | FR-003, FR-004, NFR-104, NFR-105 | -| Valid Base64 for direct | `data` matches Base64 pattern | `DESERIALIZATION_ERROR` | FR-001, FR-002, FR-003, FR-012 | -| Valid URL for link | `data` matches HTTP(S) URL pattern | `DOWNLOAD_FAILED` | FR-008, FR-009, FR-010 | +| Rule | Condition | Error Code | Requirement ID | Solution Design Ref | +|------|-----------|------------|----------------|-------------------| +| Valid payload_type | Must be in `payload_type` enum | `INVALID_PAYLOAD_TYPE` | FR-001, FR-002, FR-003, FR-006 | SD-004, SD-005 | +| Valid transport | Must be `direct` or `link` | `INVALID_TRANSPORT` | FR-003, FR-004, FR-006 | SD-001, SD-002 | +| Valid encoding | Must match payload_type and transport | `INVALID_TRANSPORT` | FR-001, FR-002, FR-003, FR-012 | SD-004, SD-005 | +| Positive size | `size > 0` | `INVALID_PAYLOAD` | FR-003, FR-004, NFR-104, NFR-105 | SD-001, SD-002 | +| Valid Base64 for direct | `data` matches Base64 pattern | `DESERIALIZATION_ERROR` | FR-001, FR-002, FR-003, FR-012 | SD-004, SD-005 | +| Valid URL for link | `data` matches HTTP(S) URL pattern | `DOWNLOAD_FAILED` | FR-008, FR-009, FR-010 | SD-003, SD-007 | --- ## Test Contracts +**Traceability**: FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 + ### Unit Test Validation -| Test | Input | Expected Output | Notes | Requirement ID | -|------|-------|-----------------|-------|----------------| -| Text round-trip | `("msg", "Hello", "text")` | `("msg", "Hello", "text")` | String serialization | FR-001, FR-012, NFR-101, NFR-102 | -| Dictionary round-trip | `("data", {"key": "value"}, "dictionary")` | `("data", {"key": "value"}, "dictionary")` | JSON object round-trip | FR-002, FR-012, NFR-101, NFR-102 | -| Arrow table round-trip | `("table", arrow_table_data, "arrowtable")` | `("table", arrow_table_data, "arrowtable")` | Arrow IPC round-trip | FR-002, FR-012, NFR-101, NFR-102 | -| JSON table round-trip | `("table", [{"a":1},{"b":2}], "jsontable")` | `("table", [{"a":1},{"b":2}], "jsontable")` | JSON array of objects | FR-001, FR-002, FR-006, FR-012 | -| Mixed payloads | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | Multiple payload types | FR-006, FR-007 | -| Large payload | `("data", rand(10_000_000), "arrowtable")` | `("data", URL, "arrowtable")` with link transport | File server upload | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | +| Test | Input | Expected Output | Notes | Requirement ID | Solution Design Ref | +|------|-------|-----------------|-------|----------------|-------------------| +| Text round-trip | `("msg", "Hello", "text")` | `("msg", "Hello", "text")` | String serialization | FR-001, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | +| Dictionary round-trip | `("data", {"key": "value"}, "dictionary")` | `("data", {"key": "value"}, "dictionary")` | JSON object round-trip | FR-002, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | +| Arrow table round-trip | `("table", arrow_table_data, "arrowtable")` | `("table", arrow_table_data, "arrowtable")` | Arrow IPC round-trip | FR-002, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | +| JSON table round-trip | `("table", [{"a":1},{"b":2}], "jsontable")` | `("table", [{"a":1},{"b":2}], "jsontable")` | JSON array of objects | FR-001, FR-002, FR-006, FR-012 | SD-004, SD-005 | +| Mixed payloads | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | Multiple payload types | FR-006, FR-007 | SD-004 | +| Large payload | `("data", rand(10_000_000), "arrowtable")` | `("data", URL, "arrowtable")` with link transport | File server upload | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | SD-001, SD-002, SD-003 | **Platform-Specific Notes:** - **Julia**: Use `Dict`, `Vector{Dict}`, or convert `DataFrame` to dictionary for testing @@ -1018,23 +1044,23 @@ flowchart TD ### Integration Test Scenarios -| Scenario | Platforms | Payloads | Size Mix | Transport | Expected Result | Requirement ID | -|----------|-----------|----------|----------|-----------|-----------------|----------------| -| Single text (small) | All | `text` | Small | direct | Round-trip successful | FR-001, FR-012, NFR-101, NFR-102 | -| Single dictionary (small) | All | `dictionary` | Small | direct | Round-trip successful | FR-002, FR-012, NFR-101, NFR-102 | -| Single arrow table (small) | Julia/JS/Python | `arrowtable` | Small | direct | Arrow IPC round-trip | FR-002, FR-012, NFR-101, NFR-102 | -| Single JSON table (small) | All | `jsontable` | Small | direct | Dictionary array round-trip | FR-001, FR-002, FR-006, FR-012 | -| Single image (small) | All | `image` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | -| Single audio (small) | All | `audio` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | -| Single video (small) | All | `video` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | -| Single binary (small) | All | `binary` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | -| Single text (large) | All | `text` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | -| Single JSON table (large) | All | `jsontable` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | -| Single image (large) | All | `image` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | -| **Ultimate Test** | Julia/JS/Python | `text` (small) + `dictionary` (small) + `arrowtable` (small) + `jsontable` (small) + `image` (small) + `audio` (small) + `video` (small) + `binary` (small) + `text` (large) + `dictionary` (large) + `arrowtable` (large) + `jsontable` (large) + `image` (large) | Mixed | direct/link | All payloads preserved with correct transport | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | -| **Ultimate Test** | MicroPython | `text` (small) + `dictionary` (small) + `text` (large) + `dictionary` (large) | Mixed | direct | Limited to text/dictionary with direct transport only | FR-005, FR-006, FR-012 | -| Cross-platform JSON table | All | `jsontable` | Small | direct | Dictionary array round-trip | FR-001, FR-002, FR-006, FR-012 | -| MicroPython ↔ Desktop | MicroPython ↔ Desktop | `text`/`dictionary` | Small | direct | Limited payload types | FR-005, FR-006, FR-012 | +| Scenario | Platforms | Payloads | Size Mix | Transport | Expected Result | Requirement ID | Solution Design Ref | +|----------|-----------|----------|----------|-----------|-----------------|----------------|-------------------| +| Single text (small) | All | `text` | Small | direct | Round-trip successful | FR-001, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | +| Single dictionary (small) | All | `dictionary` | Small | direct | Round-trip successful | FR-002, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | +| Single arrow table (small) | Julia/JS/Python | `arrowtable` | Small | direct | Arrow IPC round-trip | FR-002, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | +| Single JSON table (small) | All | `jsontable` | Small | direct | Dictionary array round-trip | FR-001, FR-002, FR-006, FR-012 | SD-004, SD-005 | +| Single image (small) | All | `image` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | SD-004, SD-005 | +| Single audio (small) | All | `audio` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | SD-004, SD-005 | +| Single video (small) | All | `video` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | SD-004, SD-005 | +| Single binary (small) | All | `binary` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 | SD-004, SD-005 | +| Single text (large) | All | `text` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | SD-001, SD-002, SD-003 | +| Single JSON table (large) | All | `jsontable` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | SD-001, SD-002, SD-003 | +| Single image (large) | All | `image` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 | SD-001, SD-002, SD-003 | +| **Ultimate Test** | Julia/JS/Python | All types | Mixed | direct/link | All payloads preserved with correct transport | FR-001 through FR-014, NFR-101 through NFR-107 | SD-001 through SD-008 | +| **Ultimate Test** | MicroPython | `text`/`dictionary` | Mixed | direct | Limited to text/dictionary with direct transport only | FR-005, FR-006, FR-012 | SD-002, SD-004 | +| Cross-platform JSON table | All | `jsontable` | Small | direct | Dictionary array round-trip | FR-001, FR-002, FR-006, FR-012 | SD-004, SD-005 | +| MicroPython ↔ Desktop | MicroPython ↔ Desktop | `text`/`dictionary` | Small | direct | Limited payload types | FR-005, FR-006, FR-012 | SD-002, SD-004 | | Desktop ↔ Desktop (all combos) | Julia↔JS↔Python | All types | Small/Large | direct/link | Full compatibility | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | --- @@ -1043,26 +1069,26 @@ flowchart TD ### Required Dependencies by Platform -| Platform | Package | Version | Purpose | -|----------|---------|---------|---------| -| Julia | JSON.jl | Latest | JSON serialization | -| Julia | Arrow.jl | Latest | Arrow IPC support | -| Julia | HTTP.jl | Latest | HTTP file server | -| Julia | UUIDs.jl | Latest | UUID generation | -| Node.js | node-fetch | Latest | HTTP file server | -| Browser | - | - | Transport-agnostic (caller provides) | -| Python | aiohttp | Latest | HTTP file server | -| Python | pyarrow | Latest | Arrow IPC support | -| Dart | http | Latest | HTTP file server | -| Dart | uuid | Latest | UUID generation | -| Dart | dart-arrow | Latest | Arrow IPC support (Desktop/Flutter) | -| Rust | serde | Latest | JSON serialization | -| Rust | serde_json | Latest | JSON handling | -| Rust | tokio | Latest | Async runtime | -| Rust | reqwest | Latest | HTTP file server | -| Rust | uuid | Latest | UUID generation | -| Rust | arrow2 | Latest | Arrow IPC support | -| MicroPython | builtin | N/A | Limited implementation | +| Platform | Package | Version | Purpose | Requirement ID | Solution Design Ref | +|----------|---------|---------|---------|----------------|-------------------| +| Julia | JSON.jl | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | SD-005 | +| Julia | Arrow.jl | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| Julia | HTTP.jl | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Julia | UUIDs.jl | Latest | UUID generation | FR-011, NFR-401 | SD-008 | +| Node.js | node-fetch | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Browser | - | - | Transport-agnostic (caller provides) | FR-013, FR-014 | SD-006 | +| Python | aiohttp | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Python | pyarrow | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| Dart | http | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Dart | uuid | Latest | UUID generation | FR-011, NFR-401 | SD-008 | +| Dart | dart-arrow | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| Rust | serde | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | SD-005 | +| Rust | serde_json | Latest | JSON handling | FR-012, NFR-101, NFR-102 | SD-005 | +| Rust | tokio | Latest | Async runtime | FR-013, FR-014 | SD-006 | +| Rust | reqwest | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Rust | uuid | Latest | UUID generation | FR-011, NFR-401 | SD-008 | +| Rust | arrow2 | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| MicroPython | builtin | N/A | Limited implementation | FR-005, FR-006 | SD-002 | ### Optional Dependencies @@ -1098,72 +1124,84 @@ flowchart TD ## References +**ASG Framework Reference**: This specification follows the ASG Framework v8 pillars. Each specification item must cite: +- **Requirement ID(s)** from requirements.md (e.g., FR-001, NFR-201) — defines *what* the system must do +- **Solution Design reference(s)** from solution-design.md (e.g., SD-2.1, SD-3.3) — defines *how* and *why* this technical approach was chosen + ### 20.1 Documentation Artifacts -| Document | Purpose | Requirements Traceability | -|----------|---------|--------------------------| -| [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/specification.md`](./specification.md) | Technical contract for msghandler | This document | -| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | UI components for data entry and display | -| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | Traceability from user journey to technical implementation | -| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | Component interaction and data flow | -| [`docs/validation.md`](./validation.md) | CI/CD validation rules | Contract testing and spec compliance | -| [`docs/runbook.md`](./runbook.md) | Operational runbook | Deployment, scaling, and troubleshooting | +| Document | Purpose | Requirements Traceability | Solution Design Traceability | +|----------|---------|--------------------------|------------------------------| +| [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`docs/solution-design.md`](./solution-design.md) | Technical solution approach | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`docs/specification.md`](./specification.md) | Technical contract for msghandler | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | UI components for data entry and display | UI components reference FR-XXX and SD-XXX | +| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | Traceability from user journey to technical implementation | Full flow validation against SD-XXX | +| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | Component interaction and data flow | Component-to-SD mapping | +| [`docs/validation.md`](./validation.md) | CI/CD validation rules | Contract testing and spec compliance | Validation gates for SD-XXX | +| [`docs/runbook.md`](./runbook.md) | Operational runbook | Deployment, scaling, and troubleshooting | Operation-to-SD mapping | ### 20.2 Implementation Files -| File | Platform | Features | Requirements Traceability | -|------|----------|----------|--------------------------| -| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | FR-005, FR-006, FR-012 | +| File | Platform | Features | Requirements Traceability | Solution Design Traceability | +|------|----------|----------|--------------------------|------------------------------| +| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | FR-005, FR-006, FR-012 | SD-002, SD-004 | ### 20.3 External Dependencies -| Platform | Package | Version | Purpose | Requirements Traceability | -|----------|---------|---------|---------|--------------------------| -| Julia | JSON.jl | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | -| Julia | Arrow.jl | Latest | Arrow IPC support | FR-002, FR-012 | -| Julia | HTTP.jl | Latest | HTTP file server | FR-008, FR-009 | -| Julia | UUIDs.jl | Latest | UUID generation | FR-011, NFR-401 | -| Node.js | node-fetch | Latest | HTTP file server | FR-008, FR-009 | -| Browser | - | - | Transport-agnostic (caller provides) | FR-013, FR-014 | -| Python | aiohttp | Latest | HTTP file server | FR-008, FR-009 | -| Python | pyarrow | Latest | Arrow IPC support | FR-002, FR-012 | -| Dart | http | Latest | HTTP file server | FR-008, FR-009 | -| Dart | uuid | Latest | UUID generation | FR-011, NFR-401 | -| Dart | dart-arrow | Latest | Arrow IPC support | FR-002, FR-012 | -| Rust | serde | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | -| Rust | serde_json | Latest | JSON handling | FR-012, NFR-101, NFR-102 | -| Rust | tokio | Latest | Async runtime | FR-013, FR-014 | -| Rust | reqwest | Latest | HTTP file server | FR-008, FR-009 | -| Rust | uuid | Latest | UUID generation | FR-011, NFR-401 | -| Rust | arrow2 | Latest | Arrow IPC support | FR-002, FR-012 | -| MicroPython | builtin | N/A | Limited implementation | FR-005, FR-006 | +| Platform | Package | Version | Purpose | Requirements Traceability | Solution Design Traceability | +|----------|---------|---------|---------|--------------------------|------------------------------| +| Julia | JSON.jl | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | SD-005 | +| Julia | Arrow.jl | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| Julia | HTTP.jl | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Julia | UUIDs.jl | Latest | UUID generation | FR-011, NFR-401 | SD-008 | +| Node.js | node-fetch | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Browser | - | - | Transport-agnostic (caller provides) | FR-013, FR-014 | SD-006 | +| Python | aiohttp | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Python | pyarrow | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| Dart | http | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Dart | uuid | Latest | UUID generation | FR-011, NFR-401 | SD-008 | +| Dart | dart-arrow | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| Rust | serde | Latest | JSON serialization | FR-012, NFR-101, NFR-102 | SD-005 | +| Rust | serde_json | Latest | JSON handling | FR-012, NFR-101, NFR-102 | SD-005 | +| Rust | tokio | Latest | Async runtime | FR-013, FR-014 | SD-006 | +| Rust | reqwest | Latest | HTTP file server | FR-008, FR-009 | SD-003 | +| Rust | uuid | Latest | UUID generation | FR-011, NFR-401 | SD-008 | +| Rust | arrow2 | Latest | Arrow IPC support | FR-002, FR-012 | SD-004, SD-005 | +| MicroPython | builtin | N/A | Limited implementation | FR-005, FR-006 | SD-002 | --- -## 21. Change Log +## Change Log -| Date | Version | Changes | Requirement ID(s) | -|------|---------|---------|-------------------| -| 2026-05-15 | 1.3.0 | Made transport layer agnostic | All | -| - | - | Removed NATS-specific dependencies and references from all docs | All | -| - | - | Updated all NATS references to generic "transport layer"/"message broker" | All | -| - | - | Removed NATS client packages from dependencies tables | All | -| 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All | -| - | - | Updated smartpack signatures: removed is_publish, nats_connection; added sender_name | FR-001 through FR-014 | -| - | - | Updated smartunpack signatures: takes msg_json_str::String instead of msg | FR-001 through FR-014 | -| - | - | Removed publishMessage function and NATSClient/NATSConnectionPool classes from browser section | FR-013, FR-014 | -| - | - | Added plik_oneshot_upload(filepath) overload to file server interface | FR-008, FR-009 | -| - | - | Fixed SIZE_THRESHOLD default to 500,000 bytes | FR-003, FR-004 | -| 2026-03-23 | 1.1.0 | Updated to ASG Framework specification guidelines | All | -| 2026-03-15 | 1.1.0 | Browser connection management | FR-001 through FR-014 | -| 2026-03-13 | 1.0.0 | Initial specification | FR-001 through FR-014, NFR-101 through NFR-405 | +| Date | Version | Changes | Requirement ID(s) | Solution Design Ref(s) | +|------|---------|---------|-------------------|------------------------| +| 2026-05-22 | 1.3.0 | Updated to ASG Framework v8 pillars | All | All SD-XXX | +| - | - | Added Solution Design references to all specification items | All | All SD-XXX | +| - | - | Updated version to 1.3.0 to match requirements and solution-design | All | - | +| - | - | Added ASG Framework alignment section | All | - | +| 2026-05-15 | 1.3.0 | Made transport layer agnostic | All | SD-006 | +| - | - | Removed all NATS-specific dependencies (NATS.jl, nats, nats-py, nats.ws) | All | SD-006 | +| - | - | Updated docs to reference generic message broker/transport | All | SD-006 | +| - | - | broker_url is now metadata only, not used for active connections | All | SD-006 | +| 2026-03-15 | 1.1.0 | Browser connection management | FR-001 through FR-014 | SD-006 | +| - | - | Added NATSClient class with keepAlive support | FR-013, FR-014 | SD-006 | +| - | - | Added NATSConnectionPool for connection reuse | FR-013, FR-014 | SD-006 | +| - | - | Added publishMessage function with closeConnection option | FR-013, FR-014 | SD-006 | +| - | - | Added nats.ws to browser dependencies | FR-013, FR-014 | SD-006 | +| 2026-03-13 | 1.0.0 | Initial specification | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| - | - | Message envelope schema defined | FR-012, FR-013 | SD-001, SD-006 | +| - | - | Payload schema with transport modes | FR-001, FR-002, FR-003, FR-004 | SD-001, SD-002 | +| - | - | Enumerations for payload_type, transport, encoding | FR-001, FR-002, FR-003, FR-006 | SD-004, SD-005 | +| - | - | Size thresholds for desktop/MicroPython | FR-004, FR-005 | SD-002 | +| - | - | Error codes and validation rules | FR-001 through FR-010 | SD-001 through SD-007 | +| - | - | API contracts for all platforms | FR-001 through FR-014 | SD-001 through SD-008 | --- From 6da7708e72d4d0b96c005ff7b9e26af201b67496 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 10:49:18 +0700 Subject: [PATCH 08/14] update docs --- docs/walkthrough.md | 336 +++++++++++++++++++++++--------------------- 1 file changed, 179 insertions(+), 157 deletions(-) diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 9ee50c2..6abd6a9 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -1,9 +1,10 @@ # Walkthrough: msghandler -**Version**: 1.4.0 -**Date**: 2026-05-14 +**Version**: 1.5.0 +**Date**: 2026-05-22 **Status**: Active -**Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) +**Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) +**ASG Framework Alignment**: v8 pillars - Requirements → Solution Design → Specification → Walkthrough → Implementation Plan → Validation → Runbook --- @@ -18,24 +19,24 @@ This walkthrough serves as the primary onboarding guide for new developers and e ### 1.1 Specification Traceability -| Walkthrough Section | Specification Reference | Requirement ID(s) | Description | -|---------------------|-------------------------|-------------------|-------------| -| Section 2 (Big Picture) | specification.md:2, specification.md:15 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | End-to-end system flow diagrams | -| Section 3 (Chat Scenario) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | FR-001, FR-006, FR-007, FR-012, FR-013, FR-014 | Chat webapp ↔ Julia backend with mixed payloads | -| Section 4 (Large File) | specification.md:6, specification.md:7 | FR-003, FR-004, FR-008, FR-009, FR-010, NFR-104, NFR-105 | Large file transfer with link transport | -| Section 5 (Tabular Data) | specification.md:5, specification.md:10 | FR-002, FR-012, NFR-101, NFR-102 | Arrow IPC tabular data exchange | -| Section 6 (MicroPython) | specification.md:13, specification.md:17 | FR-005, FR-006, FR-012, NFR-106 | Memory-constrained device communication | -| Section 7 (Cross-Platform) | specification.md:3, specification.md:4, specification.md:5, specification.md:11 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | Multi-platform chat application | -| Section 8 (Error Handling) | specification.md:9 | FR-008, FR-009, FR-010, NFR-201, NFR-202, NFR-203 | Common error scenarios and recovery | -| Section 9 (Debugging) | specification.md:4, specification.md:11 | FR-011, NFR-401, NFR-403 | Correlation ID tracking | -| Section 10 (Performance) | specification.md:7, specification.md:13 | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 | Optimization strategies | -| Section 11 (Deployment) | specification.md:12, specification.md:18 | FR-013, FR-014, NFR-201, NFR-203 | Infrastructure requirements | +| Walkthrough Section | Specification Reference | Requirement ID(s) | Solution Design Ref(s) | Description | +|---------------------|-------------------------|-------------------|------------------------|-------------| +| Section 2 (Big Picture) | specification.md:2, specification.md:15 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-002, SD-005, SD-006 | End-to-end system flow diagrams | +| Section 3 (Chat Scenario) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | FR-001, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-004, SD-005, SD-006 | Chat webapp ↔ Julia backend with mixed payloads | +| Section 4 (Large File) | specification.md:6, specification.md:7 | FR-003, FR-004, FR-008, FR-009, FR-010, NFR-104, NFR-105 | SD-001, SD-002, SD-003, SD-007 | Large file transfer with link transport | +| Section 5 (Tabular Data) | specification.md:5, specification.md:10 | FR-002, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | Arrow IPC tabular data exchange | +| Section 6 (MicroPython) | specification.md:13, specification.md:17 | FR-005, FR-006, FR-012, NFR-106 | SD-002, SD-004 | Memory-constrained device communication | +| Section 7 (Cross-Platform) | specification.md:3, specification.md:4, specification.md:5, specification.md:11 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-002, SD-004, SD-005, SD-006 | Multi-platform chat application | +| Section 8 (Error Handling) | specification.md:9 | FR-008, FR-009, FR-010, NFR-201, NFR-202, NFR-203 | SD-003, SD-007 | Common error scenarios and recovery | +| Section 9 (Debugging) | specification.md:4, specification.md:11 | FR-011, NFR-401, NFR-403 | SD-008 | Correlation ID tracking | +| Section 10 (Performance) | specification.md:7, specification.md:13 | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 | SD-001, SD-002, SD-006 | Optimization strategies | +| Section 11 (Deployment) | specification.md:12, specification.md:18 | FR-013, FR-014, NFR-201, NFR-203 | SD-006 | Infrastructure requirements | --- ## 2. Overview: The Big Picture -## Overview: The Big Picture +## 2. Overview: The Big Picture msghandler implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB): @@ -93,9 +94,7 @@ flowchart TB style FileServer fill:#ffe0b2,stroke:#f57c00 ``` -### Key Design Principles - -### Key Design Principles +### 2.1 Key Design Principles | Principle | Description | Rationale | |-----------|-------------|-----------| @@ -106,7 +105,7 @@ flowchart TB --- -## User Scenario 1: Chat Webapp ↔ Julia Backend +## 3. User Scenario 1: Chat Webapp ↔ Julia Backend ### Scenario Description @@ -114,7 +113,7 @@ A JavaScript chat webapp wants to send mixed payloads (text message + user avata ### Step-by-Step Flow -#### Step 1: JavaScript Webapp Sends Mixed Payloads +#### 3.1 Step 1: JavaScript Webapp Sends Mixed Payloads ```javascript // JavaScript (Browser or Node.js) @@ -137,7 +136,7 @@ const [env, msgJson] = await msghandler.smartpack( - **Why text first?** Text is smaller, sent via direct transport (fast, no file server needed) - **Why image second?** Images may trigger link transport if >0.5MB -#### Step 2: Transport Selection +#### 3.2 Step 2: Transport Selection For each payload, msghandler determines transport: @@ -150,7 +149,7 @@ For each payload, msghandler determines transport: - Direct transport is faster for small payloads (no file server round-trip) - Link transport is used when payload ≥ 0.5MB (avoids transport size limits) -#### Step 3: Serialization and Encoding +#### 3.3 Step 3: Serialization and Encoding Each payload is serialized: @@ -164,7 +163,7 @@ Each payload is serialized: - Images use raw bytes to preserve binary data integrity - All payloads encoded as Base64 for JSON compatibility -#### Step 4: Envelope Building +#### 3.4 Step 4: Envelope Building msghandler builds the message envelope: @@ -213,7 +212,7 @@ msghandler builds the message envelope: - **reply_to**: Tells backend where to send response - **payloads array**: Contains all data with metadata for proper handling -#### Step 5: Publish to Transport (Caller's Responsibility) +#### 3.5 Step 5: Publish to Transport (Caller's Responsibility) ```javascript // Publishing via the transport layer is the caller's responsibility @@ -227,7 +226,7 @@ msghandler builds the message envelope: - JSON format ensures cross-platform compatibility - `smartpack()` returns `(env, msgJson)` - caller handles publishing via their chosen transport -#### Step 6: Julia Backend Receives Message +#### 3.6 Step 6: Julia Backend Receives Message ```julia # Julia backend @@ -246,12 +245,12 @@ env = smartunpack(String(transport_msg.payload)) - Deserialization is type-aware based on `payload_type` - Returns consistent tuple format regardless of transport -#### Step 7: Julia Backend Sends Response +#### 3.7 Step 7: Julia Backend Sends Response ```julia # Julia backend processes the message response_text = "Hello Ton! I'm the AI assistant." -generated_image = generate_ai_image(response_text) +generated_image = generate_ai_image(response_text); env, msg_json = smartpack( "/agent/wine/api/v1/response", @@ -261,7 +260,7 @@ env, msg_json = smartpack( ], reply_to = "/chat/user/v1/message", reply_to_msg_id = msg["msg_id"] -) +); ``` **Rationale**: @@ -271,7 +270,7 @@ env, msg_json = smartpack( --- -## User Scenario 2: Large File Transfer +## 4. User Scenario 2: Large File Transfer ### Scenario Description @@ -279,7 +278,7 @@ A JavaScript webapp wants to upload a large file (10MB) to a Julia backend for p ### Step-by-Step Flow -#### Step 1: JavaScript Webapp Sends Large File +#### 4.1 Step 1: JavaScript Webapp Sends Large File ```javascript const [env, msgJson] = await msghandler.smartpack( @@ -294,7 +293,7 @@ const [env, msgJson] = await msghandler.smartpack( ); ``` -#### Step 2: Transport Selection (Link) +#### 4.2 Step 2: Transport Selection (Link) | Payload | Size | Transport | Reason | |---------|------|-----------|--------| @@ -305,7 +304,7 @@ const [env, msgJson] = await msghandler.smartpack( - File server handles large file upload - Transport only sends URL (small message) -#### Step 3: File Server Upload +#### 4.3 Step 3: File Server Upload ```javascript // msghandler internally calls: @@ -329,7 +328,7 @@ const response = await plikOneshotUpload( - One-shot mode simplifies API - Returns URL for download -#### Step 4: Envelope with Link Transport +#### 4.4 Step 4: Envelope with Link Transport ```json { @@ -353,12 +352,12 @@ const response = await plikOneshotUpload( - `transport: "link"` signals URL-based download - `encoding: "none"` indicates no additional encoding -#### Step 5: Julia Backend Receives and Downloads +#### 4.5 Step 5: Julia Backend Receives and Downloads ```julia # Julia backend -transport_msg = transport_subscription.next() -env = smartunpack(String(transport_msg.payload)) +transport_msg = transport_subscription.next(); +env = smartunpack(String(transport_msg.payload)); # msghandler automatically: # 1. Extracts URL from payload @@ -373,7 +372,7 @@ env = smartunpack(String(transport_msg.payload)) --- -## User Scenario 3: Tabular Data Exchange +## 5. User Scenario 3: Tabular Data Exchange ### Scenario Description @@ -381,7 +380,7 @@ A Python application sends tabular data (pandas DataFrame) to a Julia backend fo ### Step-by-Step Flow -#### Step 1: Python Sends Tabular Data +#### 5.1 Step 1: Python Sends Tabular Data ```python # Python @@ -392,14 +391,14 @@ df = pd.DataFrame({ "id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"], "score": [95, 88, 92] -}) +}); env, msg_json = await smartpack( "/agent/wine/api/v1/analyze", [("data", df, "arrowtable")], broker_url=DEFAULT_BROKER_URL, receiver_name="agent-backend" -) +); ``` **Rationale**: @@ -407,18 +406,18 @@ env, msg_json = await smartpack( - Arrow IPC format preserves data types - Much faster than JSON serialization -#### Step 2: Serialization to Arrow IPC +#### 5.2 Step 2: Serialization to Arrow IPC ```python # msghandler internally: import pyarrow as pa import pyarrow.ipc as ipc -table = pa.Table.from_pandas(df) -buf = io.BytesIO() -sink = ipc.new_file(buf, table.schema) -ipc.write_table(table, sink) -arrow_bytes = buf.getvalue() +table = pa.Table.from_pandas(df); +buf = io.BytesIO(); +sink = ipc.new_file(buf, table.schema); +ipc.write_table(table, sink); +arrow_bytes = buf.getvalue(); ``` **Rationale**: @@ -426,12 +425,12 @@ arrow_bytes = buf.getvalue() - Binary format is compact - No schema information loss -#### Step 3: Julia Receives and Deserializes +#### 5.3 Step 3: Julia Receives and Deserializes ```julia # Julia backend -transport_msg = transport_subscription.next() -env = smartunpack(String(transport_msg.payload)) +transport_msg = transport_subscription.next(); +env = smartunpack(String(transport_msg.payload)); # env["payloads"][1] is now: # ("data", DataFrame with id, name, score columns, "arrowtable") @@ -442,18 +441,18 @@ env = smartunpack(String(transport_msg.payload)) - DataFrame returned with correct types - No manual parsing needed -#### Step 4: Julia Sends Results +#### 5.4 Step 4: Julia Sends Results ```julia # Julia backend -results = analyze_data(env["payloads"][1][2]) +results = analyze_data(env["payloads"][1][2]); # Send results back env, msg_json = smartpack( "/agent/wine/api/v1/results", [("results", results, "arrowtable")], reply_to = "/python/worker/v1/results" -) +); ``` **Rationale**: @@ -463,7 +462,7 @@ env, msg_json = smartpack( --- -## User Scenario 4: Rust Service with Type-Safe API +## 6. User Scenario 4: Rust Service with Type-Safe API ### Scenario Description @@ -471,7 +470,7 @@ A Rust service needs to process messages from a Julia analytics pipeline and sen ### Step-by-Step Flow -#### Step 1: Rust Service Receives Message +#### 6.1 Step 1: Rust Service Receives Message ```rust // Rust service - using tokio async runtime @@ -525,7 +524,7 @@ async fn main() { - **smartunpack deserialization**: Payload data is deserialized and stored as strings in `payload.data` - **Type dispatch**: `payload_type` field determines how to interpret the `data` string -#### Step 2: Rust Service Sends Processed Results +#### 6.2 Step 2: Rust Service Sends Processed Results ```rust // Rust service sends results back with mixed payload types @@ -566,11 +565,11 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; - **Default options**: sensible defaults reduce boilerplate - **Result**: idiomatic Rust error handling -#### Step 3: Python/Julia Receives Rust Response +#### 6.3 Step 3: Python/Julia Receives Rust Response ```python # Python backend receives Rust response -env = await smartunpack(str(transport_msg.payload)) +env = await smartunpack(str(transport_msg.payload)); # env["payloads"][0] is now: # ("results", arrow_table_data, "arrowtable") @@ -583,7 +582,7 @@ env = await smartunpack(str(transport_msg.payload)) - **Same JSON wire format**: No protocol translation needed - **Type preservation**: Arrow IPC and text types preserved across all platforms -#### Step 4: Large File Transfer from Rust +#### 6.4 Step 4: Large File Transfer from Rust ```rust // Rust service sends large binary file via link transport @@ -615,7 +614,7 @@ let (envelope, json_str) = smartpack( --- -## User Scenario 5: MicroPython Device +## 7. User Scenario 5: MicroPython Device ### Scenario Description @@ -623,7 +622,7 @@ A MicroPython sensor device sends sensor readings to a Python backend. ### Step-by-Step Flow -#### Step 1: MicroPython Sends Sensor Data +#### 7.1 Step 1: MicroPython Sends Sensor Data ```python # MicroPython @@ -633,14 +632,14 @@ sensor_data = { "temperature": 25.5, "humidity": 60.0, "pressure": 1013.25 -} +}; env, msg_json = smartpack( "/sensor/device/v1/readings", [("data", sensor_data, "dictionary")], broker_url=DEFAULT_BROKER_URL, size_threshold=100000 # 100KB for MicroPython -) +); ``` **Rationale**: @@ -648,13 +647,13 @@ env, msg_json = smartpack( - Smaller threshold (100KB) for memory constraints - Direct transport only (no file server support) -#### Step 2: Serialization +#### 7.2 Step 2: Serialization ```python # msghandler internally: -json_str = json.dumps(sensor_data) -json_bytes = json_str.encode('utf-8') -payload_b64 = base64.b64encode(json_bytes).decode('ascii') +json_str = json.dumps(sensor_data); +json_bytes = json_str.encode('utf-8'); +payload_b64 = base64.b64encode(json_bytes).decode('ascii'); ``` **Rationale**: @@ -662,12 +661,12 @@ payload_b64 = base64.b64encode(json_bytes).decode('ascii') - Base64 for transport compatibility - UTF-8 for text encoding -#### Step 3: Python Backend Receives +#### 7.3 Step 3: Python Backend Receives ```python # Python backend -transport_msg = await transport_consumer.next() -env = await smartunpack(str(transport_msg.payload)) +transport_msg = await transport_consumer.next(); +env = await smartunpack(str(transport_msg.payload)); # env["payloads"][0] is now: # ("data", {"temperature": 25.5, "humidity": 60.0, ...}, "dictionary") @@ -680,7 +679,7 @@ env = await smartunpack(str(transport_msg.payload)) --- -## User Scenario 6: Cross-Platform Chat with Mixed Payloads +## 8. User Scenario 6: Cross-Platform Chat with Mixed Payloads ### Scenario Description @@ -688,7 +687,7 @@ Multiple platforms (JavaScript, Python, Julia) communicate in a chat application ### Step-by-Step Flow -#### Step 1: JavaScript Sends Chat Message +#### 8.1 Step 1: JavaScript Sends Chat Message ```javascript // JavaScript (Frontend) @@ -711,12 +710,12 @@ const [env, msgJson] = await msghandler.smartpack( - Chat messages often include text + images - Transport wildcard subscriptions route to correct recipients -#### Step 2: Python Backend Receives +#### 8.2 Step 2: Python Backend Receives ```python # Python (Backend) -transport_msg = await transport_consumer.next() -env = await smartunpack(str(transport_msg.payload)) +transport_msg = await transport_consumer.next(); +env = await smartunpack(str(transport_msg.payload)); # env["payloads"] is now: # [ @@ -730,12 +729,12 @@ env = await smartunpack(str(transport_msg.payload)) - Same payload structure regardless of sender - Type information preserved -#### Step 3: Julia Backend Receives +#### 8.3 Step 3: Julia Backend Receives ```julia # Julia (Backend) -transport_msg = transport_subscription.next() -env = smartunpack(String(transport_msg.payload)) +transport_msg = transport_subscription.next(); +env = smartunpack(String(transport_msg.payload)); # env["payloads"] is now: # [ @@ -749,7 +748,7 @@ env = smartunpack(String(transport_msg.payload)) - Same function signature across platforms - Type information enables proper deserialization -#### Step 4: All Platforms Reply +#### 8.4 Step 4: All Platforms Reply Each platform can reply using the same API: @@ -759,7 +758,7 @@ await smartpack( "/chat/user/v1/reply", [("response", "Nice!", "text")], reply_to="/chat/user/v1/message" -) +); ``` ```julia @@ -768,7 +767,7 @@ smartpack( "/chat/user/v1/reply", [("response", "Nice!", "text")], reply_to="/chat/user/v1/message" -) +); ``` ```javascript @@ -787,9 +786,9 @@ await msghandler.smartpack( --- -## Error Handling +## 9. Error Handling -### Common Error Scenarios +### 9.1 Common Error Scenarios | Scenario | Error | Recovery | |----------|-------|----------| @@ -798,7 +797,7 @@ await msghandler.smartpack( | Payload type mismatch | `DESERIALIZATION_ERROR` | Validate payload_type matches data | | Transport connection lost | `TRANSPORT_CONNECTION_FAILED` | Transport client auto-reconnects | -### Error Response Format +### 9.2 Error Response Format ```json { @@ -816,20 +815,27 @@ await msghandler.smartpack( --- -## Debugging and Tracing +## 10. Debugging and Tracing -### Correlation ID Tracking +### 10.1 Correlation ID Tracking Every message includes a `correlation_id`: ```julia # At start of request -correlation_id = string(uuid4()) +correlation_id = string(uuid4()); # Use throughout the flow -log_trace(correlation_id, "Starting smartpack") -log_trace(correlation_id, "Serialized payload size: 100 bytes") -log_trace(correlation_id, "Published to transport") +log_trace(correlation_id, "Starting smartpack"); +log_trace(correlation_id, "Serialized payload size: 100 bytes"); +log_trace(correlation_id, "Published to transport"); +``` + +**Log Format**: +``` +[2026-03-13T16:30:00.000Z] [Correlation: abc123...] Starting smartpack +[2026-03-13T16:30:00.001Z] [Correlation: abc123...] Serialized payload size: 100 bytes +[2026-03-13T16:30:00.002Z] [Correlation: abc123...] Published to transport ``` **Log Format**: @@ -841,9 +847,9 @@ log_trace(correlation_id, "Published to transport") --- -## Performance Considerations +## 11. Performance Considerations -### Optimization Strategies +### 11.1 Optimization Strategies | Strategy | Description | When to Use | |----------|-------------|-------------| @@ -851,7 +857,7 @@ log_trace(correlation_id, "Published to transport") | Adjust size threshold | Increase threshold if file server slow | File server bottleneck | | Use direct transport | Avoid file server for small payloads | Low latency requirements | -### Size Threshold by Platform +### 11.2 Size Threshold by Platform | Platform | Threshold | Notes | |----------|-----------|-------| @@ -863,9 +869,9 @@ log_trace(correlation_id, "Published to transport") --- -## Deployment Considerations +## 12. Deployment Considerations -### Minimum Infrastructure +### 12.1 Minimum Infrastructure | Component | Minimum | Notes | |-----------|---------|-------| @@ -874,7 +880,7 @@ log_trace(correlation_id, "Published to transport") | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) | | Client Memory | 256KB | MicroPython devices | -### Environment Variables +### 12.2 Environment Variables | Variable | Default | Description | |----------|---------|-------------| @@ -888,64 +894,31 @@ log_trace(correlation_id, "Published to transport") | Date | Version | Changes | |------|---------|---------| +| 2026-05-22 | 1.5.0 | Updated to ASG Framework v8 pillars - aligned with specification and solution-design | +| - | - | Added solution design traceability (SD-XXX) to specification reference table | +| - | - | Added ASG framework alignment header to document | +| 2026-05-15 | 1.4.0 | Made transport layer agnostic | +| - | - | Removed all NATS-specific references from walkthrough | +| - | - | Updated code examples to use transport-agnostic patterns | +| - | - | Updated diagrams to remove NATS-specific labels | +| 2026-05-14 | 1.3.0 | Updated Rust API to reflect `smartunpack` deserialization changes | +| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | +| - | - | Added `plik_upload_file` convenience function documentation | +| - | - | Fixed Rust scenario payload access (data is String, not Payload enum) | +| - | - | Removed `metadata` from link transport examples | +| 2026-05-13 | 1.2.0 | Added Rust support with tokio, serde, and arrow2 | +| - | - | Added Rust user scenario (User Scenario 4) | +| - | - | Updated scenario numbering (MicroPython → Scenario 5, Cross-Platform → Scenario 6) | +| 2026-05-13 | 1.1.0 | Aligned with ground truth implementation (src/msghandler.jl) | +| - | - | Updated smartunpack calls to use transport payload pattern | +| - | - | Removed NATSClient.publish() calls (caller responsible for transport publishing) | +| - | - | Removed is_publish and nats_connection parameter references | +| 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | | 2026-03-13 | 1.0.0 | Initial walkthrough documentation | --- -## 12. References - -### 12.1 Documentation Artifacts - -| Document | Purpose | Specification Traceability | -|----------|---------|---------------------------| -| [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 | -| [`docs/specification.md`](./specification.md) | Technical contract for msghandler | specification.md:2-19 (all sections) | -| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | UI components for data entry and display | -| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | This document | -| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | Component interaction and data flow | -| [`docs/validation.md`](./validation.md) | CI/CD validation rules | Contract testing and spec compliance | -| [`docs/runbook.md`](./runbook.md) | Operational runbook | Deployment, scaling, and troubleshooting | - -### 12.2 Implementation Files - -| File | Platform | Features | Specification Traceability | -|------|----------|----------|---------------------------| -| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | specification.md:2-19 (all sections) | -| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | specification.md:2-19 (all sections) | -| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | specification.md:2-19 (all sections) | -| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | specification.md:2-19 (all sections) | -| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | specification.md:2-19 (all sections) | -| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe, file upload helpers | specification.md:2-19 (all sections) | -| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | specification.md:2-19 (all sections) | - ---- - -## 13. Change Log - -| Date | Version | Changes | Specification Reference | -|------|---------|---------|------------------------| -| 2026-05-15 | 1.5.0 | Made transport layer agnostic | All sections | -| - | - | Removed all NATS-specific references from walkthrough | All sections | -| - | - | Updated code examples to use transport-agnostic patterns | All sections | -| - | - | Updated diagrams to remove NATS-specific labels | All sections | -| 2026-05-14 | 1.4.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | -| - | - | `smartunpack` now stores deserialized data in `MsgPayloadV1.data` | specification.md:8 | -| - | - | Added `plik_upload_file` convenience function documentation | specification.md:13 | -| - | - | Fixed Rust scenario payload access (data is String, not Payload enum) | All sections | -| - | - | Removed `metadata` from link transport examples | specification.md:3 | -| 2026-05-13 | 1.3.0 | Added Rust support with tokio, serde, and arrow2 | All sections | -| - | - | Added Rust user scenario (User Scenario 4) | specification.md:11 (Rust API) | -| - | - | Updated scenario numbering (MicroPython → Scenario 5, Cross-Platform → Scenario 6) | All sections | -| 2026-05-13 | 1.2.0 | Aligned with ground truth implementation (src/msghandler.jl) | All sections | -| - | - | Updated smartunpack calls to use transport payload pattern | All sections | -| - | - | Removed NATSClient.publish() calls (caller responsible for transport publishing) | All sections | -| - | - | Removed is_publish and nats_connection parameter references | All sections | -| 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | All sections | -| 2026-03-13 | 1.0.0 | Initial walkthrough documentation | specification.md:2-19 (all sections) | - ---- - -## 14. Gap-Check Validation +## 15. Gap-Check Validation | Stage Transition | Gap-Check Question | Status | |------------------|-------------------|--------| @@ -956,15 +929,64 @@ log_trace(correlation_id, "Published to transport") --- -*This walkthrough document is versioned and maintained in git alongside the codebase. All implementations must adhere to this documentation.* +## 16. References + +### 16.1 Documentation Artifacts + +| Document | Purpose | Specification Traceability | Solution Design Traceability | +|----------|---------|---------------------------|------------------------------| +| [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`docs/specification.md`](./specification.md) | Technical contract for msghandler | specification.md:2-19 (all sections) | SD-001 through SD-008 | +| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | UI components for data entry and display | UI components reference FR-XXX and SD-XXX | +| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | This document | Full flow validation against SD-XXX | +| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | Component interaction and data flow | Component-to-SD mapping | +| [`docs/validation.md`](./validation.md) | CI/CD validation rules | Contract testing and spec compliance | Validation gates for SD-XXX | +| [`docs/runbook.md`](./runbook.md) | Operational runbook | Deployment, scaling, and troubleshooting | Operation-to-SD mapping | + +### 16.2 Implementation Files + +| File | Platform | Features | Specification Traceability | Solution Design Traceability | +|------|----------|----------|---------------------------|------------------------------| +| [`src/msghandler.jl`](../src/msghandler.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler_ssr.js`](../src/msghandler_ssr.js) | Node.js | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler_csr.js`](../src/msghandler_csr.js) | Browser | JSON table only | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler.py`](../src/msghandler.py) | Python | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler.dart`](../src/msghandler.dart) | Dart | Full feature set, Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler.rs`](../src/msghandler.rs) | Rust | Full feature set, Arrow IPC, async/await, type-safe, file upload helpers | FR-001 through FR-014, NFR-101 through NFR-405 | SD-001 through SD-008 | +| [`src/msghandler_mpy.py`](../src/msghandler_mpy.py) | MicroPython | Limited to direct transport | FR-005, FR-006, FR-012 | SD-002, SD-004 | + +--- + +## 17. Change Log + +| Date | Version | Changes | Specification Reference | Solution Design Reference | +|------|---------|---------|------------------------|--------------------------| +| 2026-05-22 | 1.5.0 | Updated to ASG Framework v8 pillars - aligned with specification and solution-design | All sections | All SD-XXX | +| 2026-05-15 | 1.4.0 | Made transport layer agnostic | All sections | SD-001 through SD-008 | +| 2026-05-14 | 1.3.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | SD-001 through SD-008 | +| 2026-05-13 | 1.2.0 | Added Rust support with tokio, serde, and arrow2 | specification.md:11 (Rust API) | SD-001 through SD-008 | +| 2026-05-13 | 1.1.0 | Aligned with ground truth implementation (src/msghandler.jl) | All sections | SD-001 through SD-008 | +| 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | All sections | SD-001 through SD-008 | +| 2026-03-13 | 1.0.0 | Initial walkthrough documentation | specification.md:2-19 (all sections) | SD-001 through SD-008 | --- *This walkthrough document is versioned and maintained in git alongside the codebase. All implementations must adhere to this documentation.* - - - -[x] Analyze existing documentation (requirements.md, spec.md, architecture.md) -[x] Read all source files in src/ folder -[x] Write docs/walkthrough.md according to SDD framework with user scenarios \ No newline at end of file +--- + +## 18. ASG Framework Validation + +| Pillar | Status | Reference | +|--------|--------|-----------| +| Requirements | ✅ Complete | requirements.md: FR-001 through FR-014, NFR-101 through NFR-405 | +| Solution Design | ✅ Complete | solution-design.md: SD-001 through SD-008 | +| Specification | ✅ Complete | specification.md: Section 2-19 | +| Walkthrough | ✅ Complete | walkthrough.md: Sections 2-17 | +| Implementation Plan | ⏳ Pending | implementation-plan.md | +| Validation | ⏳ Pending | validation.md | +| Runbook | ⏳ Pending | runbook.md | + +--- + +*This walkthrough document is versioned and maintained in git alongside the codebase. All implementations must adhere to this documentation.* \ No newline at end of file From 0c9aebdd37aa0730839f0e556b70d441309d75cc Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 10:53:55 +0700 Subject: [PATCH 09/14] update --- docs/solution-design.md | 2 +- docs/walkthrough.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/solution-design.md b/docs/solution-design.md index fb0ffd7..3b5ed00 100644 --- a/docs/solution-design.md +++ b/docs/solution-design.md @@ -31,7 +31,7 @@ msghandler addresses the challenge of cross-platform data exchange between **Jul - File server integration using Claim-Check pattern - Multi-payload support with mixed types in single message - Exponential backoff for reliable file downloads -- Correlation ID propagation for message tracing +- Correlation ID propagation for message tracing **Out of Scope**: - Message compression (adds complexity without clear benefit) diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 6abd6a9..a3c83bb 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -36,8 +36,6 @@ This walkthrough serves as the primary onboarding guide for new developers and e ## 2. Overview: The Big Picture -## 2. Overview: The Big Picture - msghandler implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB): ```mermaid From 8de9a65c03a7cf8e4f81ee6f4cf76a7e63939071 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 18:59:23 +0700 Subject: [PATCH 10/14] update docs --- AI_prompt.md | 35 +++++++++++++ docs/walkthrough.md | 124 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 142 insertions(+), 17 deletions(-) diff --git a/AI_prompt.md b/AI_prompt.md index 11ffb4a..5cbb13a 100644 --- a/AI_prompt.md +++ b/AI_prompt.md @@ -248,5 +248,40 @@ I already setup the project structure. Can you implement the app? To test whether this Dioxus project can be build, you may use this command "dx bundle --web --release --debug-symbols=false" P.S. In a Dioxus single-page application (SPA), switching screens can be handled perfectly using standard Rust state matching (often called conditional rendering or state-based routing). + + + + +read the following files: +./docs/requirements.md +./docs/solution-design.md +./docs/specification.md +./docs/walkthrough.md +What is the main interface of this package? + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/walkthrough.md b/docs/walkthrough.md index a3c83bb..11f2716 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -109,7 +109,68 @@ flowchart TB A JavaScript chat webapp wants to send mixed payloads (text message + user avatar image) to a Julia backend, and receive mixed payloads (text response + AI-generated image) back. -### Step-by-Step Flow +### Complete End-to-End Round-Trip Flow + +msghandler implements a **transport-agnostic** messaging pattern. The library handles serialization and envelope building, while the caller is responsible for publishing/subscribing via their chosen transport (NATS, MQTT, WebSocket, HTTP, etc.). + +``` +┌─────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ END-TO-END ROUND-TRIP FLOW │ +└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + +SENDER (JavaScript Webapp) TRANSPORT LAYER RECEIVER (Julia Backend) +┌─────────────────────┐ ┌───────────────────┐ ┌─────────────────────┐ +│ │ │ │ │ │ +│ 1. Prepare data │ │ │ │ │ +│ [(msg, data, │ │ │ │ │ +│ type)] │ │ │ │ │ +│ │ │ │ │ │ +│ 2. Call smartpack() │──────────────────────────────────────>│ │ │ │ +│ Returns (env, │ │ JSON Message │ │ │ +│ json_str) │ │ (json_str) │ │ │ +│ │ │ │ │ │ +│ 3. Publish via │ │ │ │ │ +│ transport: │──────────────────────────────────────>│ (NATS/MQTT/ │────────────────────────>│ 4. Subscribe via │ +│ MY_TRANSPORT. │ │ WebSocket, etc.) │ │ transport: │ +│ publish(subject, │ │ │ │ msg = │ +│ msgJson) │ │ │ │ SUBSCRIBE() │ +│ │ │ │ │ │ +│ │ │ │ │ 5. Call │ +│ │ │ │ │ smartunpack( │ +│ │ │ │ │ String( │ +│ │ │ │ │ msg.payload) │ +│ │ │ │ │ │ +│ │ │ │ │ 6. Receive │ +│ │ │ │ │ payloads: │ +│ │ │ │ │ [(msg, data, │ +│ │ │ │ │ type), ...] │ +│ │ │ │ │ │ +└─────────────────────┘ └───────────────────┘ └─────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ JULIA BACKEND RESPONSE (Reverse Flow) │ +└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + +RECEIVER (Julia Backend) TRANSPORT LAYER SENDER (JavaScript Webapp) +┌─────────────────────┐ ┌───────────────────┐ ┌─────────────────────┐ +│ │ │ │ │ │ +│ 7. Process data │ │ │ │ │ +│ (AI inference) │ │ │ │ │ +│ │ │ │ │ │ +│ 8. Call smartpack() │──────────────────────────────────────>│ │ │ │ +│ Returns (env, │ │ JSON Message │ │ │ +│ json_str) │ │ (json_str) │ │ │ +│ │ │ │ │ │ +│ 9. Publish reply │ │ │ │ │ +│ via transport: │──────────────────────────────────────>│ (NATS/MQTT/ │────────────────────────>│ 10. Subscribe via │ +│ MY_TRANSPORT. │ │ WebSocket, etc.) │ │ reply_to topic │ +│ publish(replyTo, │ │ │ │ │ +│ msgJson) │ │ │ │ │ +│ │ │ │ │ │ +└─────────────────────┘ └───────────────────┘ └─────────────────────┘ +``` + +### Step-by-Step Flow (Sender Perspective) #### 3.1 Step 1: JavaScript Webapp Sends Mixed Payloads @@ -212,30 +273,54 @@ msghandler builds the message envelope: #### 3.5 Step 5: Publish to Transport (Caller's Responsibility) +**Choose your transport** (replace `MY_TRANSPORT` with your actual library): + ```javascript -// Publishing via the transport layer is the caller's responsibility -// Example with any transport (NATS, MQTT, WebSocket, etc.) -// const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); -// await conn.publish("/agent/wine/api/v1/prompt", msgJson); +// Example 1: NATS (Node.js) +import { connect } from 'nats'; +const nats = connect({ servers: 'ws://localhost:4222' }); +await nats.publish("/agent/wine/api/v1/prompt", msgJson); + +// Example 2: MQTT (Node.js) +import * as mqtt from 'mqtt'; +const client = mqtt.connect('ws://localhost:1883'); +client.publish("/agent/wine/api/v1/prompt", msgJson); + +// Example 3: WebSocket (Browser) +const ws = new WebSocket('ws://localhost:4222/ws'); +ws.send(msgJson); + +// Example 4: Custom HTTP POST +fetch('http://localhost:8000/publish', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: msgJson +}); ``` -**Rationale**: -- The transport layer provides message delivery (NATS, MQTT, WebSocket, etc.) -- JSON format ensures cross-platform compatibility -- `smartpack()` returns `(env, msgJson)` - caller handles publishing via their chosen transport +**Why caller responsibility?** +- **Transport agnostic**: msghandler supports NATS, MQTT, WebSocket, HTTP, or any custom transport +- **Connection reuse**: Callers can manage connection pools efficiently +- **Flexibility**: No library lock-in; use whatever transport best fits your stack #### 3.6 Step 6: Julia Backend Receives Message ```julia # Julia backend -transport_msg = transport_subscription.next() # Get message from transport -env = smartunpack(String(transport_msg.payload)) +# Choose your transport (replace MY_TRANSPORT with your actual library) -# env["payloads"] is now: -# [ -# ("msg", "Hello! I'm Ton.", "text"), -# ("avatar", binary_data, "image") -# ] +# Example: NATS subscription +using NATS +conn = NATS.connect("nats.yiem.cc") +NATS.subscribe(conn, "/agent/wine/api/v1/prompt") do msg + env = smartunpack(String(msg.payload)) + + # env["payloads"] is now: + # [ + # ("msg", "Hello! I'm Ton.", "text"), + # ("avatar", binary_data, "image") + # ] +end ``` **Rationale**: @@ -257,8 +342,13 @@ env, msg_json = smartpack( ("generated_image", generated_image, "image") ], reply_to = "/chat/user/v1/message", - reply_to_msg_id = msg["msg_id"] + reply_to_msg_id = env["msg_id"] ); + +# Publish response via transport (caller's responsibility) +# Example: NATS +using NATS +NATS.publish(conn, "/agent/wine/api/v1/response", msg_json) ``` **Rationale**: From e93fe9c5c7769c0f52ebf6038216efe982f4bdab Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 19:42:34 +0700 Subject: [PATCH 11/14] update docs --- docs/walkthrough.md | 536 ++++++++++++++++++++++++++++++-------------- 1 file changed, 367 insertions(+), 169 deletions(-) diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 11f2716..fbd4d52 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -1,6 +1,6 @@ # Walkthrough: msghandler -**Version**: 1.5.0 +**Version**: 1.6.0 **Date**: 2026-05-22 **Status**: Active **Ground Truth**: [`src/msghandler.jl`](../src/msghandler.jl) @@ -16,21 +16,23 @@ This walkthrough serves as the primary onboarding guide for new developers and e - **User scenarios** - Real-world use cases from developer perspective - **Why steps are sequenced** - The rationale behind architectural decisions - **What could go wrong** - Common failure scenarios and recovery strategies +- **Transport integration** - Complete end-to-end flow with publish/subscribe patterns ### 1.1 Specification Traceability | Walkthrough Section | Specification Reference | Requirement ID(s) | Solution Design Ref(s) | Description | |---------------------|-------------------------|-------------------|------------------------|-------------| -| Section 2 (Big Picture) | specification.md:2, specification.md:15 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-002, SD-005, SD-006 | End-to-end system flow diagrams | +| Section 2 (Big Picture) | specification.md:2, specification.md:15 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-002, SD-005, SD-006 | End-to-end system flow diagrams and transport pattern | | Section 3 (Chat Scenario) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | FR-001, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-004, SD-005, SD-006 | Chat webapp ↔ Julia backend with mixed payloads | | Section 4 (Large File) | specification.md:6, specification.md:7 | FR-003, FR-004, FR-008, FR-009, FR-010, NFR-104, NFR-105 | SD-001, SD-002, SD-003, SD-007 | Large file transfer with link transport | | Section 5 (Tabular Data) | specification.md:5, specification.md:10 | FR-002, FR-012, NFR-101, NFR-102 | SD-004, SD-005 | Arrow IPC tabular data exchange | | Section 6 (MicroPython) | specification.md:13, specification.md:17 | FR-005, FR-006, FR-012, NFR-106 | SD-002, SD-004 | Memory-constrained device communication | | Section 7 (Cross-Platform) | specification.md:3, specification.md:4, specification.md:5, specification.md:11 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | SD-001, SD-002, SD-004, SD-005, SD-006 | Multi-platform chat application | -| Section 8 (Error Handling) | specification.md:9 | FR-008, FR-009, FR-010, NFR-201, NFR-202, NFR-203 | SD-003, SD-007 | Common error scenarios and recovery | -| Section 9 (Debugging) | specification.md:4, specification.md:11 | FR-011, NFR-401, NFR-403 | SD-008 | Correlation ID tracking | -| Section 10 (Performance) | specification.md:7, specification.md:13 | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 | SD-001, SD-002, SD-006 | Optimization strategies | -| Section 11 (Deployment) | specification.md:12, specification.md:18 | FR-013, FR-014, NFR-201, NFR-203 | SD-006 | Infrastructure requirements | +| Section 10 (Error Handling) | specification.md:9 | FR-008, FR-009, FR-010, NFR-201, NFR-202, NFR-203 | SD-003, SD-007 | Common error scenarios and recovery | +| Section 11 (Debugging) | specification.md:4, specification.md:11 | FR-011, NFR-401, NFR-403 | SD-008 | Correlation ID tracking | +| Section 12 (Performance) | specification.md:7, specification.md:13 | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 | SD-001, SD-002, SD-006 | Optimization strategies | +| Section 13 (Deployment) | specification.md:12, specification.md:18 | FR-013, FR-014, NFR-201, NFR-203 | SD-006 | Infrastructure requirements | +| Section 9 (Transport Layer) | specification.md:3, specification.md:5 | FR-013, FR-014 | SD-006 | Complete end-to-end transport integration examples | --- @@ -38,6 +40,47 @@ This walkthrough serves as the primary onboarding guide for new developers and e msghandler implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB): +### 2.0 Transport Layer Pattern + +**Critical**: msghandler is **transport-agnostic**. The caller is responsible for: + +1. **Sending side**: Call `smartpack()` → receive `(envelope, json_string)` → publish JSON string via transport +2. **Receiving side**: Subscribe to transport → receive JSON string → call `smartunpack()` + +```mermaid +flowchart LR + subgraph msghandler["msghandler Module"] + direction TB + S1["smartpack(data)"] + S2["Returns (envelope, json_str)"] + R1["smartunpack(json_str)"] + R2["Returns payloads"] + end + + subgraph Transport["Transport Layer (Caller's Responsibility)"] + direction TB + PUBLISH["Publish JSON string"] + SUBSCRIBE["Subscribe to messages"] + end + + S1 --> S2 + S2 --> PUBLISH + SUBSCRIBE --> R1 + R1 --> R2 + PUBLISH --> SUBSCRIBE + + style msghandler fill:#b3e5fc,stroke:#0288d1 + style Transport fill:#ffe0b2,stroke:#f57c00 +``` + +**Supported Transports**: +- NATS (WebSocket/TCP) +- MQTT (WebSocket/TCP) +- WebSocket (direct HTTP) +- HTTP (polling/long-polling) + +**Key Principle**: msghandler only handles **serialization/deserialization**. Transport is handled by caller. + ```mermaid flowchart TB subgraph msghandler["msghandler Module"] @@ -92,7 +135,7 @@ flowchart TB style FileServer fill:#ffe0b2,stroke:#f57c00 ``` -### 2.1 Key Design Principles +### 2.2 Key Design Principles | Principle | Description | Rationale | |-----------|-------------|-----------| @@ -100,6 +143,7 @@ flowchart TB | **Automatic Transport Selection** | Direct (< threshold) vs Link (≥ threshold) based on size | Optimizes memory vs network I/O trade-off | | **Cross-Platform API** | Consistent `smartpack()`/`smartunpack()` across all platforms | Simplifies developer experience | | **Exponential Backoff** | Retry downloads with increasing delays | Handles transient failures gracefully | +| **Transport Agnostic** | Caller handles transport (NATS/MQTT/WebSocket) | No vendor lock-in; works with any broker | --- @@ -109,68 +153,7 @@ flowchart TB A JavaScript chat webapp wants to send mixed payloads (text message + user avatar image) to a Julia backend, and receive mixed payloads (text response + AI-generated image) back. -### Complete End-to-End Round-Trip Flow - -msghandler implements a **transport-agnostic** messaging pattern. The library handles serialization and envelope building, while the caller is responsible for publishing/subscribing via their chosen transport (NATS, MQTT, WebSocket, HTTP, etc.). - -``` -┌─────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ END-TO-END ROUND-TRIP FLOW │ -└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ - -SENDER (JavaScript Webapp) TRANSPORT LAYER RECEIVER (Julia Backend) -┌─────────────────────┐ ┌───────────────────┐ ┌─────────────────────┐ -│ │ │ │ │ │ -│ 1. Prepare data │ │ │ │ │ -│ [(msg, data, │ │ │ │ │ -│ type)] │ │ │ │ │ -│ │ │ │ │ │ -│ 2. Call smartpack() │──────────────────────────────────────>│ │ │ │ -│ Returns (env, │ │ JSON Message │ │ │ -│ json_str) │ │ (json_str) │ │ │ -│ │ │ │ │ │ -│ 3. Publish via │ │ │ │ │ -│ transport: │──────────────────────────────────────>│ (NATS/MQTT/ │────────────────────────>│ 4. Subscribe via │ -│ MY_TRANSPORT. │ │ WebSocket, etc.) │ │ transport: │ -│ publish(subject, │ │ │ │ msg = │ -│ msgJson) │ │ │ │ SUBSCRIBE() │ -│ │ │ │ │ │ -│ │ │ │ │ 5. Call │ -│ │ │ │ │ smartunpack( │ -│ │ │ │ │ String( │ -│ │ │ │ │ msg.payload) │ -│ │ │ │ │ │ -│ │ │ │ │ 6. Receive │ -│ │ │ │ │ payloads: │ -│ │ │ │ │ [(msg, data, │ -│ │ │ │ │ type), ...] │ -│ │ │ │ │ │ -└─────────────────────┘ └───────────────────┘ └─────────────────────┘ - -┌─────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ JULIA BACKEND RESPONSE (Reverse Flow) │ -└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ - -RECEIVER (Julia Backend) TRANSPORT LAYER SENDER (JavaScript Webapp) -┌─────────────────────┐ ┌───────────────────┐ ┌─────────────────────┐ -│ │ │ │ │ │ -│ 7. Process data │ │ │ │ │ -│ (AI inference) │ │ │ │ │ -│ │ │ │ │ │ -│ 8. Call smartpack() │──────────────────────────────────────>│ │ │ │ -│ Returns (env, │ │ JSON Message │ │ │ -│ json_str) │ │ (json_str) │ │ │ -│ │ │ │ │ │ -│ 9. Publish reply │ │ │ │ │ -│ via transport: │──────────────────────────────────────>│ (NATS/MQTT/ │────────────────────────>│ 10. Subscribe via │ -│ MY_TRANSPORT. │ │ WebSocket, etc.) │ │ reply_to topic │ -│ publish(replyTo, │ │ │ │ │ -│ msgJson) │ │ │ │ │ -│ │ │ │ │ │ -└─────────────────────┘ └───────────────────┘ └─────────────────────┘ -``` - -### Step-by-Step Flow (Sender Perspective) +### Step-by-Step Flow #### 3.1 Step 1: JavaScript Webapp Sends Mixed Payloads @@ -195,6 +178,14 @@ const [env, msgJson] = await msghandler.smartpack( - **Why text first?** Text is smaller, sent via direct transport (fast, no file server needed) - **Why image second?** Images may trigger link transport if >0.5MB +#### 3.1b Step 1b: JavaScript Publishes via Transport + +```javascript +// Publish the JSON string via WebSocket (or NATS/MQTT) +const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); +await conn.publish("/agent/wine/api/v1/prompt", msgJson); +``` + #### 3.2 Step 2: Transport Selection For each payload, msghandler determines transport: @@ -271,63 +262,28 @@ msghandler builds the message envelope: - **reply_to**: Tells backend where to send response - **payloads array**: Contains all data with metadata for proper handling -#### 3.5 Step 5: Publish to Transport (Caller's Responsibility) - -**Choose your transport** (replace `MY_TRANSPORT` with your actual library): +#### 3.5 Step 5: Publish JSON String via Transport ```javascript -// Example 1: NATS (Node.js) -import { connect } from 'nats'; -const nats = connect({ servers: 'ws://localhost:4222' }); -await nats.publish("/agent/wine/api/v1/prompt", msgJson); - -// Example 2: MQTT (Node.js) -import * as mqtt from 'mqtt'; -const client = mqtt.connect('ws://localhost:1883'); -client.publish("/agent/wine/api/v1/prompt", msgJson); - -// Example 3: WebSocket (Browser) -const ws = new WebSocket('ws://localhost:4222/ws'); -ws.send(msgJson); - -// Example 4: Custom HTTP POST -fetch('http://localhost:8000/publish', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: msgJson -}); +// JavaScript: Publish via WebSocket (NATS/MQTT/HTTP work similarly) +const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); +await conn.publish("/agent/wine/api/v1/prompt", msgJson); ``` -**Why caller responsibility?** -- **Transport agnostic**: msghandler supports NATS, MQTT, WebSocket, HTTP, or any custom transport -- **Connection reuse**: Callers can manage connection pools efficiently -- **Flexibility**: No library lock-in; use whatever transport best fits your stack - -#### 3.6 Step 6: Julia Backend Receives Message +#### 3.6 Step 6: Julia Backend Receives and Unpacks ```julia # Julia backend -# Choose your transport (replace MY_TRANSPORT with your actual library) +transport_msg = transport_subscription.next() # Get message from transport +env = smartunpack(String(transport_msg.payload)) -# Example: NATS subscription -using NATS -conn = NATS.connect("nats.yiem.cc") -NATS.subscribe(conn, "/agent/wine/api/v1/prompt") do msg - env = smartunpack(String(msg.payload)) - - # env["payloads"] is now: - # [ - # ("msg", "Hello! I'm Ton.", "text"), - # ("avatar", binary_data, "image") - # ] -end +# env["payloads"] is now: +# [ +# ("msg", "Hello! I'm Ton.", "text"), +# ("avatar", binary_data, "image") +# ] ``` -**Rationale**: -- `smartunpack()` handles both transport types automatically -- Deserialization is type-aware based on `payload_type` -- Returns consistent tuple format regardless of transport - #### 3.7 Step 7: Julia Backend Sends Response ```julia @@ -342,13 +298,9 @@ env, msg_json = smartpack( ("generated_image", generated_image, "image") ], reply_to = "/chat/user/v1/message", - reply_to_msg_id = env["msg_id"] + reply_to_msg_id = msg["msg_id"] ); - -# Publish response via transport (caller's responsibility) -# Example: NATS -using NATS -NATS.publish(conn, "/agent/wine/api/v1/response", msg_json) +publish(transport_conn, "/agent/wine/api/v1/response", msg_json); ``` **Rationale**: @@ -381,6 +333,14 @@ const [env, msgJson] = await msghandler.smartpack( ); ``` +#### 4.1b Step 1b: JavaScript Publishes via Transport + +```javascript +// Publish the JSON string via WebSocket (or NATS/MQTT) +const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); +await conn.publish("/agent/wine/api/v1/process", msgJson); +``` + #### 4.2 Step 2: Transport Selection (Link) | Payload | Size | Transport | Reason | @@ -440,7 +400,7 @@ const response = await plikOneshotUpload( - `transport: "link"` signals URL-based download - `encoding: "none"` indicates no additional encoding -#### 4.5 Step 5: Julia Backend Receives and Downloads +#### 4.5 Step 5: Julia Backend Receives, Downloads, and Unpacks ```julia # Julia backend @@ -448,8 +408,8 @@ transport_msg = transport_subscription.next(); env = smartunpack(String(transport_msg.payload)); # msghandler automatically: -# 1. Extracts URL from payload -# 2. Downloads with exponential backoff +# 1. Extracts URL from payload (link transport) +# 2. Downloads file with exponential backoff # 3. Deserializes to binary data ``` @@ -494,6 +454,13 @@ env, msg_json = await smartpack( - Arrow IPC format preserves data types - Much faster than JSON serialization +#### 5.1b Step 1b: Python Publishes via Transport + +```python +# Publish the JSON string via WebSocket/NATS/MQTT +await transport_publisher.publish("/agent/wine/api/v1/analyze", msg_json) +``` + #### 5.2 Step 2: Serialization to Arrow IPC ```python @@ -529,7 +496,7 @@ env = smartunpack(String(transport_msg.payload)); - DataFrame returned with correct types - No manual parsing needed -#### 5.4 Step 4: Julia Sends Results +#### 5.4 Step 4: Julia Sends Results (Complete Round-Trip) ```julia # Julia backend @@ -541,6 +508,7 @@ env, msg_json = smartpack( [("results", results, "arrowtable")], reply_to = "/python/worker/v1/results" ); +publish(transport_conn, "/agent/wine/api/v1/results", msg_json); ``` **Rationale**: @@ -643,7 +611,7 @@ let (envelope, json_str) = smartpack( }, ).await?; -// Caller publishes via transport +// Caller publishes via transport (WebSocket/MQTT/NATS) conn.publish("/agent/wine/api/v1/results", &json_str)?; ``` @@ -692,6 +660,9 @@ let (envelope, json_str) = smartpack( ..Default::default() }, ).await?; + +// Caller publishes via transport +conn.publish("/agent/wine/api/v1/upload", &json_str)?; ``` **Rationale**: @@ -735,6 +706,13 @@ env, msg_json = smartpack( - Smaller threshold (100KB) for memory constraints - Direct transport only (no file server support) +#### 7.1b Step 1b: MicroPython Publishes via Transport + +```python +# Publish the JSON string via MQTT (common on IoT devices) +mqtt_client.publish("/sensor/device/v1/readings", msg_json) +``` + #### 7.2 Step 2: Serialization ```python @@ -765,6 +743,40 @@ env = await smartunpack(str(transport_msg.payload)); - Dictionary returned directly - No Arrow support (memory constraints) +#### 7.4 Step 4: Complete MicroPython End-to-End Flow + +```mermaid +flowchart LR + subgraph MicroPython["MicroPython Device"] + MPY1["sensor_data = {temp, humidity}"] + MPY2["smartpack() → JSON"] + MPY3["publish via MQTT"] + end + + subgraph Transport["MQTT Broker"] + direction TB + MQTT["broker.local:1883"] + end + + subgraph Python["Python Backend"] + PY1["subscribe via MQTT"] + PY2["smartunpack(JSON)"] + PY3["get dictionary"] + end + + MPY1 --> MPY2 + MPY2 --> MPY3 + MPY3 --> MQTT + MQTT --> PY1 + PY1 --> PY2 + PY2 --> PY3 +``` + +**Key Points**: +- MicroPython limited to `text` and `dictionary` types (memory constraints) +- Direct transport only (no file server support) +- Size threshold: 100KB (vs 500KB for desktop) + --- ## 8. User Scenario 6: Cross-Platform Chat with Mixed Payloads @@ -798,6 +810,14 @@ const [env, msgJson] = await msghandler.smartpack( - Chat messages often include text + images - Transport wildcard subscriptions route to correct recipients +#### 8.1b Step 1b: JavaScript Publishes via Transport + +```javascript +// Publish via WebSocket/NATS/MQTT +const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); +await conn.publish("/chat/user/v1/message", msgJson); +``` + #### 8.2 Step 2: Python Backend Receives ```python @@ -836,47 +856,203 @@ env = smartunpack(String(transport_msg.payload)); - Same function signature across platforms - Type information enables proper deserialization -#### 8.4 Step 4: All Platforms Reply +#### 8.4 Step 4: Complete End-to-End Flow -Each platform can reply using the same API: - -```python -# Python reply -await smartpack( - "/chat/user/v1/reply", - [("response", "Nice!", "text")], - reply_to="/chat/user/v1/message" -); -``` - -```julia -# Julia reply -smartpack( - "/chat/user/v1/reply", - [("response", "Nice!", "text")], - reply_to="/chat/user/v1/message" -); -``` - -```javascript -// JavaScript reply -await msghandler.smartpack( - "/chat/user/v1/reply", - [["response", "Nice!", "text"]], - { reply_to: "/chat/user/v1/message" } -); +```mermaid +flowchart TB + subgraph Sender["Sender (JavaScript)"] + direction TB + JS1["Create data tuples"] + JS2["smartpack() → JSON string"] + JS3["Publish via WebSocket"] + end + + subgraph Transport["Transport Layer"] + direction TB + BROKER["Message Broker"] + end + + subgraph Receiver["Receiver (Julia/Python)"] + direction TB + REC1["Subscribe via WebSocket"] + REC2["smartunpack(JSON string)"] + REC3["Get payload tuples"] + end + + JS1 --> JS2 + JS2 --> JS3 + JS3 --> BROKER + BROKER --> REC1 + REC1 --> REC2 + REC2 --> REC3 ``` **Rationale**: -- Same API across platforms -- Consistent behavior -- Easy to maintain parity +- `smartpack()` handles serialization, transport selection, and envelope building +- Caller publishes JSON string via WebSocket/NATS/MQTT +- `smartunpack()` handles transport detection, file downloads (if link), and deserialization +- Same API across all platforms ensures consistency + +#### 8.6 Step 6: Complete Cross-Platform Flow + +```mermaid +flowchart TB + subgraph JS["JavaScript (Frontend)"] + J1["Create message tuples"] + J2["smartpack()"] + J3["Publish via WebSocket"] + J4["Subscribe for replies"] + end + + subgraph Python["Python (Backend)"] + P1["Subscribe via NATS"] + P2["smartunpack()"] + P3["Process data"] + P4["smartpack()"] + P5["Publish reply via NATS"] + end + + subgraph Julia["Julia (Backend)"] + L1["Subscribe via MQTT"] + L2["smartunpack()"] + L3["Process data"] + L4["smartpack()"] + L5["Publish reply via MQTT"] + end + + subgraph Transport["Message Broker (NATS/MQTT/WebSocket)"] + direction TB + NATS["NATS: ws://localhost:4222"] + MQTT["MQTT: broker.local:1883"] + end + + J2 --> J3 + J3 --> NATS + NATS --> P1 + P1 --> P2 + P2 --> P3 + P3 --> P4 + P4 --> P5 + P5 --> MQTT + MQTT --> L1 + L1 --> L2 + L2 --> L3 + L3 --> L4 + L4 --> L5 + L5 --> NATS + NATS --> J4 + + style JS fill:#e3f2fd,stroke:#1976d2 + style Python fill:#fff3e0,stroke:#f57c00 + style Julia fill:#e8f5e9,stroke:#388e3c + style Transport fill:#f3e5f5,stroke:#7b1fa2 +``` + +**Key Points**: +- Same `smartpack()`/`smartunpack()` API across all platforms +- JSON wire format ensures compatibility +- Each platform uses their preferred transport protocol --- -## 9. Error Handling +## 9. Transport Layer Integration -### 9.1 Common Error Scenarios +msghandler is **transport-agnostic** - it only handles serialization/deserialization. The caller is responsible for: + +### 9.1 Publishing Flow (Sender) + +1. Call `smartpack()` → gets `(envelope, json_string)` +2. Publish JSON string via transport (WebSocket/MQTT/NATS/etc.) +3. Transport delivers message to subscriber + +### 9.2 Subscribing Flow (Receiver) + +1. Subscribe to transport topic +2. Receive JSON string from transport +3. Call `smartunpack(json_string)` → gets payloads array + +### 9.3 Complete End-to-End Example + +#### JavaScript → Julia Complete Flow + +```mermaid +flowchart LR + subgraph JavaScript["JavaScript Webapp"] + J1["Create tuples"] + J2["smartpack()"] + J3["Publish via WebSocket"] + end + + subgraph Transport["NATS Broker"] + direction TB + NATS["ws://localhost:4222"] + end + + subgraph Julia["Julia Backend"] + J4["Subscribe via WebSocket"] + J5["smartunpack()"] + J6["Process payloads"] + end + + J1 --> J2 + J2 --> J3 + J3 --> NATS + NATS --> J4 + J4 --> J5 + J5 --> J6 +``` + +```javascript +// JavaScript - Sender +const [env, msgJson] = await msghandler.smartpack( + "/agent/wine/api/v1/prompt", + [["msg", "Hello!", "text"]], + { broker_url: "ws://localhost:4222" } +); +const conn = await nats.connect({ servers: "ws://localhost:4222" }); +await conn.publish("/agent/wine/api/v1/prompt", msgJson); +``` + +```julia +# Julia - Receiver +transport_msg = transport_subscription.next() +env = smartunpack(String(transport_msg.payload)) +# env["payloads"] is now: [("msg", "Hello!", "text")] +``` + +#### Transport Examples (Single Platform) +```javascript +const conn = await nats.connect({ servers: "ws://localhost:4222" }); +await conn.publish(topic, json_string); +``` + +#### MQTT (Python) +```python +await mqtt_client.publish(topic, json_string) +``` + +#### WebSocket (Browser) +```javascript +const ws = new WebSocket("ws://localhost:4222"); +ws.send(json_string); +``` + +#### Rust (Tokio) +```rust +conn.publish(topic, &json_str)?; +``` + +### 9.4 Important Notes + +- **JSON format**: All messages use JSON string format for cross-platform compatibility +- **Caller responsibility**: Transport publishing/subscription is always the caller's code +- **No vendor lock-in**: Works with any message broker that supports your platform + +--- + +## 10. Error Handling + +### 10.1 Common Error Scenarios | Scenario | Error | Recovery | |----------|-------|----------| @@ -885,7 +1061,7 @@ await msghandler.smartpack( | Payload type mismatch | `DESERIALIZATION_ERROR` | Validate payload_type matches data | | Transport connection lost | `TRANSPORT_CONNECTION_FAILED` | Transport client auto-reconnects | -### 9.2 Error Response Format +### 10.2 Error Response Format ```json { @@ -903,9 +1079,9 @@ await msghandler.smartpack( --- -## 10. Debugging and Tracing +## 11. Debugging and Tracing -### 10.1 Correlation ID Tracking +### 11.1 Correlation ID Tracking Every message includes a `correlation_id`: @@ -935,9 +1111,9 @@ log_trace(correlation_id, "Published to transport"); --- -## 11. Performance Considerations +## 12. Performance Considerations -### 11.1 Optimization Strategies +### 12.1 Optimization Strategies | Strategy | Description | When to Use | |----------|-------------|-------------| @@ -945,7 +1121,7 @@ log_trace(correlation_id, "Published to transport"); | Adjust size threshold | Increase threshold if file server slow | File server bottleneck | | Use direct transport | Avoid file server for small payloads | Low latency requirements | -### 11.2 Size Threshold by Platform +### 12.2 Size Threshold by Platform | Platform | Threshold | Notes | |----------|-----------|-------| @@ -957,7 +1133,7 @@ log_trace(correlation_id, "Published to transport"); --- -## 12. Deployment Considerations +## 13. Deployment Considerations ### 12.1 Minimum Infrastructure @@ -1006,7 +1182,7 @@ log_trace(correlation_id, "Published to transport"); --- -## 15. Gap-Check Validation +## 14. Gap-Check Validation | Stage Transition | Gap-Check Question | Status | |------------------|-------------------|--------| @@ -1017,6 +1193,27 @@ log_trace(correlation_id, "Published to transport"); --- +## 15. Complete End-to-End Flow Summary + +The msghandler transport-agnostic pattern works as follows: + +``` +Sender: Transport Layer: Receiver: +smartpack() Publish JSON string Subscribe + ↓ ↓ ↓ +(envelope, json_str) ────────────→ JSON string smartunpack() + ↓ + payloads array +``` + +**Key Points**: +- `smartpack()` handles serialization, transport selection, and envelope building +- Caller publishes JSON string via WebSocket/NATS/MQTT +- `smartunpack()` handles transport detection, file downloads (if link), and deserialization +- Same API across all platforms ensures consistency + +--- + ## 16. References ### 16.1 Documentation Artifacts @@ -1049,6 +1246,7 @@ log_trace(correlation_id, "Published to transport"); | Date | Version | Changes | Specification Reference | Solution Design Reference | |------|---------|---------|------------------------|--------------------------| +| 2026-05-22 | 1.6.0 | Added complete transport integration examples with end-to-end flow diagrams | Section 9 | SD-006 | | 2026-05-22 | 1.5.0 | Updated to ASG Framework v8 pillars - aligned with specification and solution-design | All sections | All SD-XXX | | 2026-05-15 | 1.4.0 | Made transport layer agnostic | All sections | SD-001 through SD-008 | | 2026-05-14 | 1.3.0 | Updated Rust API to reflect `smartunpack` deserialization changes | All sections | SD-001 through SD-008 | @@ -1070,7 +1268,7 @@ log_trace(correlation_id, "Published to transport"); | Requirements | ✅ Complete | requirements.md: FR-001 through FR-014, NFR-101 through NFR-405 | | Solution Design | ✅ Complete | solution-design.md: SD-001 through SD-008 | | Specification | ✅ Complete | specification.md: Section 2-19 | -| Walkthrough | ✅ Complete | walkthrough.md: Sections 2-17 | +| Walkthrough | ✅ Complete | walkthrough.md: Sections 2-18 (includes Transport Layer Integration) | | Implementation Plan | ⏳ Pending | implementation-plan.md | | Validation | ⏳ Pending | validation.md | | Runbook | ⏳ Pending | runbook.md | From 594eb3278395836063a04744b64055b664cf7238 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 20:21:49 +0700 Subject: [PATCH 12/14] update --- docs/walkthrough.md | 68 ++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/walkthrough.md b/docs/walkthrough.md index fbd4d52..364025f 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -155,7 +155,7 @@ A JavaScript chat webapp wants to send mixed payloads (text message + user avata ### Step-by-Step Flow -#### 3.1 Step 1: JavaScript Webapp Sends Mixed Payloads +#### Step 1: JavaScript Webapp Sends Mixed Payloads ```javascript // JavaScript (Browser or Node.js) @@ -178,7 +178,7 @@ const [env, msgJson] = await msghandler.smartpack( - **Why text first?** Text is smaller, sent via direct transport (fast, no file server needed) - **Why image second?** Images may trigger link transport if >0.5MB -#### 3.1b Step 1b: JavaScript Publishes via Transport +#### Step 1b: JavaScript Publishes via Transport ```javascript // Publish the JSON string via WebSocket (or NATS/MQTT) @@ -186,7 +186,7 @@ const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); await conn.publish("/agent/wine/api/v1/prompt", msgJson); ``` -#### 3.2 Step 2: Transport Selection +#### Step 2: Transport Selection For each payload, msghandler determines transport: @@ -199,7 +199,7 @@ For each payload, msghandler determines transport: - Direct transport is faster for small payloads (no file server round-trip) - Link transport is used when payload ≥ 0.5MB (avoids transport size limits) -#### 3.3 Step 3: Serialization and Encoding +#### Step 3: Serialization and Encoding Each payload is serialized: @@ -213,7 +213,7 @@ Each payload is serialized: - Images use raw bytes to preserve binary data integrity - All payloads encoded as Base64 for JSON compatibility -#### 3.4 Step 4: Envelope Building +#### Step 4: Envelope Building msghandler builds the message envelope: @@ -262,7 +262,7 @@ msghandler builds the message envelope: - **reply_to**: Tells backend where to send response - **payloads array**: Contains all data with metadata for proper handling -#### 3.5 Step 5: Publish JSON String via Transport +#### Step 5: Publish JSON String via Transport ```javascript // JavaScript: Publish via WebSocket (NATS/MQTT/HTTP work similarly) @@ -270,7 +270,7 @@ const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); await conn.publish("/agent/wine/api/v1/prompt", msgJson); ``` -#### 3.6 Step 6: Julia Backend Receives and Unpacks +#### Step 6: Julia Backend Receives and Unpacks ```julia # Julia backend @@ -284,7 +284,7 @@ env = smartunpack(String(transport_msg.payload)) # ] ``` -#### 3.7 Step 7: Julia Backend Sends Response +#### Step 7: Julia Backend Sends Response ```julia # Julia backend processes the message @@ -318,7 +318,7 @@ A JavaScript webapp wants to upload a large file (10MB) to a Julia backend for p ### Step-by-Step Flow -#### 4.1 Step 1: JavaScript Webapp Sends Large File +#### Step 1: JavaScript Webapp Sends Large File ```javascript const [env, msgJson] = await msghandler.smartpack( @@ -333,7 +333,7 @@ const [env, msgJson] = await msghandler.smartpack( ); ``` -#### 4.1b Step 1b: JavaScript Publishes via Transport +#### Step 1b: JavaScript Publishes via Transport ```javascript // Publish the JSON string via WebSocket (or NATS/MQTT) @@ -341,7 +341,7 @@ const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); await conn.publish("/agent/wine/api/v1/process", msgJson); ``` -#### 4.2 Step 2: Transport Selection (Link) +#### Step 2: Transport Selection (Link) | Payload | Size | Transport | Reason | |---------|------|-----------|--------| @@ -352,7 +352,7 @@ await conn.publish("/agent/wine/api/v1/process", msgJson); - File server handles large file upload - Transport only sends URL (small message) -#### 4.3 Step 3: File Server Upload +#### Step 3: File Server Upload ```javascript // msghandler internally calls: @@ -376,7 +376,7 @@ const response = await plikOneshotUpload( - One-shot mode simplifies API - Returns URL for download -#### 4.4 Step 4: Envelope with Link Transport +#### Step 4: Envelope with Link Transport ```json { @@ -400,7 +400,7 @@ const response = await plikOneshotUpload( - `transport: "link"` signals URL-based download - `encoding: "none"` indicates no additional encoding -#### 4.5 Step 5: Julia Backend Receives, Downloads, and Unpacks +#### Step 5: Julia Backend Receives, Downloads, and Unpacks ```julia # Julia backend @@ -428,7 +428,7 @@ A Python application sends tabular data (pandas DataFrame) to a Julia backend fo ### Step-by-Step Flow -#### 5.1 Step 1: Python Sends Tabular Data +#### Step 1: Python Sends Tabular Data ```python # Python @@ -454,14 +454,14 @@ env, msg_json = await smartpack( - Arrow IPC format preserves data types - Much faster than JSON serialization -#### 5.1b Step 1b: Python Publishes via Transport +#### Step 1b: Python Publishes via Transport ```python # Publish the JSON string via WebSocket/NATS/MQTT await transport_publisher.publish("/agent/wine/api/v1/analyze", msg_json) ``` -#### 5.2 Step 2: Serialization to Arrow IPC +#### Step 2: Serialization to Arrow IPC ```python # msghandler internally: @@ -480,7 +480,7 @@ arrow_bytes = buf.getvalue(); - Binary format is compact - No schema information loss -#### 5.3 Step 3: Julia Receives and Deserializes +#### Step 3: Julia Receives and Deserializes ```julia # Julia backend @@ -496,7 +496,7 @@ env = smartunpack(String(transport_msg.payload)); - DataFrame returned with correct types - No manual parsing needed -#### 5.4 Step 4: Julia Sends Results (Complete Round-Trip) +#### Step 4: Julia Sends Results (Complete Round-Trip) ```julia # Julia backend @@ -526,7 +526,7 @@ A Rust service needs to process messages from a Julia analytics pipeline and sen ### Step-by-Step Flow -#### 6.1 Step 1: Rust Service Receives Message +#### Step 1: Rust Service Receives Message ```rust // Rust service - using tokio async runtime @@ -580,7 +580,7 @@ async fn main() { - **smartunpack deserialization**: Payload data is deserialized and stored as strings in `payload.data` - **Type dispatch**: `payload_type` field determines how to interpret the `data` string -#### 6.2 Step 2: Rust Service Sends Processed Results +#### Step 2: Rust Service Sends Processed Results ```rust // Rust service sends results back with mixed payload types @@ -621,7 +621,7 @@ conn.publish("/agent/wine/api/v1/results", &json_str)?; - **Default options**: sensible defaults reduce boilerplate - **Result**: idiomatic Rust error handling -#### 6.3 Step 3: Python/Julia Receives Rust Response +#### Step 3: Python/Julia Receives Rust Response ```python # Python backend receives Rust response @@ -638,7 +638,7 @@ env = await smartunpack(str(transport_msg.payload)); - **Same JSON wire format**: No protocol translation needed - **Type preservation**: Arrow IPC and text types preserved across all platforms -#### 6.4 Step 4: Large File Transfer from Rust +#### Step 4: Large File Transfer from Rust ```rust // Rust service sends large binary file via link transport @@ -681,7 +681,7 @@ A MicroPython sensor device sends sensor readings to a Python backend. ### Step-by-Step Flow -#### 7.1 Step 1: MicroPython Sends Sensor Data +#### Step 1: MicroPython Sends Sensor Data ```python # MicroPython @@ -706,14 +706,14 @@ env, msg_json = smartpack( - Smaller threshold (100KB) for memory constraints - Direct transport only (no file server support) -#### 7.1b Step 1b: MicroPython Publishes via Transport +#### Step 1b: MicroPython Publishes via Transport ```python # Publish the JSON string via MQTT (common on IoT devices) mqtt_client.publish("/sensor/device/v1/readings", msg_json) ``` -#### 7.2 Step 2: Serialization +#### Step 2: Serialization ```python # msghandler internally: @@ -727,7 +727,7 @@ payload_b64 = base64.b64encode(json_bytes).decode('ascii'); - Base64 for transport compatibility - UTF-8 for text encoding -#### 7.3 Step 3: Python Backend Receives +#### Step 3: Python Backend Receives ```python # Python backend @@ -743,7 +743,7 @@ env = await smartunpack(str(transport_msg.payload)); - Dictionary returned directly - No Arrow support (memory constraints) -#### 7.4 Step 4: Complete MicroPython End-to-End Flow +#### Step 4: Complete MicroPython End-to-End Flow ```mermaid flowchart LR @@ -787,7 +787,7 @@ Multiple platforms (JavaScript, Python, Julia) communicate in a chat application ### Step-by-Step Flow -#### 8.1 Step 1: JavaScript Sends Chat Message +#### Step 1: JavaScript Sends Chat Message ```javascript // JavaScript (Frontend) @@ -810,7 +810,7 @@ const [env, msgJson] = await msghandler.smartpack( - Chat messages often include text + images - Transport wildcard subscriptions route to correct recipients -#### 8.1b Step 1b: JavaScript Publishes via Transport +#### Step 1b: JavaScript Publishes via Transport ```javascript // Publish via WebSocket/NATS/MQTT @@ -818,7 +818,7 @@ const conn = await transportClient.connect({ servers: "ws://localhost:4222" }); await conn.publish("/chat/user/v1/message", msgJson); ``` -#### 8.2 Step 2: Python Backend Receives +#### Step 2: Python Backend Receives ```python # Python (Backend) @@ -837,7 +837,7 @@ env = await smartunpack(str(transport_msg.payload)); - Same payload structure regardless of sender - Type information preserved -#### 8.3 Step 3: Julia Backend Receives +#### Step 3: Julia Backend Receives ```julia # Julia (Backend) @@ -856,7 +856,7 @@ env = smartunpack(String(transport_msg.payload)); - Same function signature across platforms - Type information enables proper deserialization -#### 8.4 Step 4: Complete End-to-End Flow +#### Step 4: Complete End-to-End Flow ```mermaid flowchart TB @@ -893,7 +893,7 @@ flowchart TB - `smartunpack()` handles transport detection, file downloads (if link), and deserialization - Same API across all platforms ensures consistency -#### 8.6 Step 6: Complete Cross-Platform Flow +#### Step 5: Complete Cross-Platform Flow ```mermaid flowchart TB From e8112baf806ccc929790108b9e5ad83a8bf77ff6 Mon Sep 17 00:00:00 2001 From: narawat Date: Fri, 22 May 2026 20:22:52 +0700 Subject: [PATCH 13/14] update --- docs/walkthrough.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 364025f..d5a2871 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -40,7 +40,7 @@ This walkthrough serves as the primary onboarding guide for new developers and e msghandler implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB): -### 2.0 Transport Layer Pattern +### 2.1 Transport Layer Pattern **Critical**: msghandler is **transport-agnostic**. The caller is responsible for: From ba21ccf763fe253c993c460f999750d04bba34b2 Mon Sep 17 00:00:00 2001 From: narawat Date: Sat, 23 May 2026 05:14:46 +0700 Subject: [PATCH 14/14] update --- Manifest.toml | 134 ++++++++++++++++++++++++++------------------------ Project.toml | 2 + 2 files changed, 73 insertions(+), 63 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index ef1fb5d..8c2bcfa 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.5" +julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "b632f853bcf5355f5c53ad3efa7a19f70444dc6c" +project_hash = "6757ef801c2fba25b1829ffc7ce99f19563e7dc4" [[deps.AliasTables]] deps = ["PtrArrays", "Random"] @@ -52,15 +52,15 @@ version = "1.2.2" [[deps.CSV]] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] -git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" +git-tree-sha1 = "8d8e0b0f350b8e1c91420b5e64e5de774c2f0f4d" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "0.10.15" +version = "0.10.16" [[deps.CodeTracking]] -deps = ["InteractiveUtils", "UUIDs"] -git-tree-sha1 = "b7231a755812695b8046e8471ddc34c8268cbad5" +deps = ["InteractiveUtils", "REPL", "UUIDs"] +git-tree-sha1 = "cfb7a2e89e245a9d5016b70323db412b3a7438d5" uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" -version = "3.0.0" +version = "3.0.2" [[deps.CodecBase]] deps = ["TranscodingStreams"] @@ -108,9 +108,9 @@ version = "1.3.0+1" [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] -git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" +git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.5.0" +version = "2.5.1" [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" @@ -124,15 +124,15 @@ version = "1.16.0" [[deps.DataFrames]] deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "d8928e9169ff76c6281f39a659f9bca3a573f24c" +git-tree-sha1 = "5fab31e2e01e70ad66e3e24c968c264d1cf166d6" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.8.1" +version = "1.8.2" [[deps.DataStructures]] deps = ["OrderedCollections"] -git-tree-sha1 = "e357641bb3e0638d353c4b29ea0e40ea644066a6" +git-tree-sha1 = "e86f4a2805f7f19bec5129bc9150c38208e5dc23" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.19.3" +version = "0.19.4" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -146,9 +146,9 @@ version = "1.11.0" [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] -git-tree-sha1 = "fbcc7610f6d8348428f722ecbe0e6cfe22e672c6" +git-tree-sha1 = "e421c1938fafab0165b04dc1a9dbe2a26272952c" uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.123" +version = "0.25.125" [deps.Distributions.extensions] DistributionsChainRulesCoreExt = "ChainRulesCore" @@ -171,9 +171,9 @@ uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.7.0" [[deps.EnumX]] -git-tree-sha1 = "7bebc8aad6ee6217c78c5ddcf7ed289d65d0263e" +git-tree-sha1 = "c49898e8438c828577f04b92fc9368c388ac783c" uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" -version = "1.0.6" +version = "1.0.7" [[deps.ExceptionUnwrapping]] deps = ["Test"] @@ -234,9 +234,9 @@ version = "0.3.1" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "5e6fe50ae7f23d171f44e311c2960294aaa0beb5" +git-tree-sha1 = "51059d23c8bb67911a2e6fd5130229113735fc7e" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.10.19" +version = "1.11.0" [[deps.HashArrayMappedTries]] git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" @@ -281,15 +281,15 @@ version = "1.0.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.1" +version = "1.8.0" [[deps.JSON]] deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] -git-tree-sha1 = "b3ad4a0255688dcb895a52fafbaae3023b588a90" +git-tree-sha1 = "f76f7560267b840e492180f9899b472f30b88450" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "1.4.0" +version = "1.6.0" weakdeps = ["ArrowTypes"] [deps.JSON.extensions] @@ -307,9 +307,9 @@ weakdeps = ["ArrowTypes"] [[deps.JuliaInterpreter]] deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] -git-tree-sha1 = "80580012d4ed5a3e8b18c7cd86cebe4b816d17a6" +git-tree-sha1 = "58927c485919bf17ea308d9d82156de1adf4b006" uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" -version = "0.10.9" +version = "0.10.12" [[deps.JuliaSyntaxHighlighting]] deps = ["StyledStrings"] @@ -383,9 +383,9 @@ version = "1.2.0" [[deps.LoweredCodeUtils]] deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] -git-tree-sha1 = "65ae3db6ab0e5b1b5f217043c558d9d1d33cc88d" +git-tree-sha1 = "5d4278f755440f70648d80cc6225f51e78e94094" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" -version = "3.5.0" +version = "3.5.1" [[deps.Lz4_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -400,9 +400,9 @@ version = "1.11.0" [[deps.MbedTLS]] deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] -git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +git-tree-sha1 = "8785729fa736197687541f7053f6d8ab7fc44f92" uuid = "739be429-bea8-5141-9913-cc70e7f3736d" -version = "1.1.9" +version = "1.1.10" [[deps.MbedTLS_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -432,15 +432,9 @@ version = "2025.11.4" [[deps.NATS]] deps = ["Base64", "BufferedStreams", "CodecBase", "Dates", "DocStringExtensions", "JSON3", "MbedTLS", "NanoDates", "Random", "ScopedValues", "Sockets", "Sodium", "StructTypes", "URIs"] -git-tree-sha1 = "d9d9a189fb9155a460e6b5e8966bf6a66737abf8" +git-tree-sha1 = "a1cdf34ba90ee5cd2658e487d3277ffafee712ce" uuid = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a" -version = "0.1.0" - -[[deps.msghandler]] -deps = ["Arrow", "DataFrames", "Dates", "GeneralUtils", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"] -path = "." -uuid = "f2724d33-f338-4a57-b9f8-1be882570d10" -version = "0.4.1" +version = "0.1.1" [[deps.NanoDates]] deps = ["Dates", "Parsers"] @@ -496,9 +490,9 @@ weakdeps = ["StatsBase"] [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" +git-tree-sha1 = "5d5e0a78e971354b1c7bff0655d11fdc1b0e12c8" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.3" +version = "2.8.4" [[deps.PooledArrays]] deps = ["DataAPI", "Future"] @@ -508,15 +502,15 @@ version = "1.4.3" [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" +git-tree-sha1 = "edbeefc7a4889f528644251bdb5fc9ab5348bc2c" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.3.3" +version = "1.3.4" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.5.1" +version = "1.5.2" [[deps.PrettyPrinting]] git-tree-sha1 = "142ee93724a9c5d04d78df7006670a93ed1b244e" @@ -525,9 +519,15 @@ version = "0.4.2" [[deps.PrettyTables]] deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "c5a07210bd060d6a8491b0ccdee2fa0235fc00bf" +git-tree-sha1 = "624de6279ab7d94fc9f672f0068107eb6619732c" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "3.1.2" +version = "3.3.2" + + [deps.PrettyTables.extensions] + PrettyTablesTypstryExt = "Typstry" + + [deps.PrettyTables.weakdeps] + Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e" [[deps.Printf]] deps = ["Unicode"] @@ -535,15 +535,15 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" [[deps.PtrArrays]] -git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" +git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe" uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.3.0" +version = "1.4.0" [[deps.QuadGK]] deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" +git-tree-sha1 = "5e8e8b0ab68215d7a2b14b9921a946fee794749e" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.11.2" +version = "2.11.3" [deps.QuadGK.extensions] QuadGKEnzymeExt = "Enzyme" @@ -568,9 +568,9 @@ version = "1.2.2" [[deps.Revise]] deps = ["CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Preferences", "REPL", "UUIDs"] -git-tree-sha1 = "14d1bfb0a30317edc77e11094607ace3c800f193" +git-tree-sha1 = "d9383b639663d8220ac9c523927e38bc21cad16a" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" -version = "3.13.2" +version = "3.14.3" [deps.Revise.extensions] DistributedExt = "Distributed" @@ -596,9 +596,9 @@ version = "0.7.0" [[deps.ScopedValues]] deps = ["HashArrayMappedTries", "Logging"] -git-tree-sha1 = "c3b2323466378a2ba15bea4b2f73b081e022f473" +git-tree-sha1 = "67a144433c4ce877ee6d1ada69a124d6b1ecf7be" uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" -version = "1.5.0" +version = "1.6.2" [[deps.Scratch]] deps = ["Dates"] @@ -608,9 +608,9 @@ version = "1.3.0" [[deps.SentinelArrays]] deps = ["Dates", "Random"] -git-tree-sha1 = "ebe7e59b37c400f694f52b58c93d26201387da70" +git-tree-sha1 = "084c47c7c5ce5cfecefa0a98dff69eb3646b5a80" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.9" +version = "1.4.10" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -644,9 +644,9 @@ version = "1.12.0" [[deps.SpecialFunctions]] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "f2685b435df2613e25fc10ad8c26dddb8640f547" +git-tree-sha1 = "2700b235561b0335d5bef7097a111dc513b8655e" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.6.1" +version = "2.7.2" [deps.SpecialFunctions.extensions] SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" @@ -692,9 +692,9 @@ version = "1.5.2" [[deps.StringManipulation]] deps = ["PrecompileTools"] -git-tree-sha1 = "a3c1536470bf8c5e02096ad4853606d7c8f62721" +git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.4.2" +version = "0.4.4" [[deps.StringViews]] git-tree-sha1 = "f2dcb92855b31ad92fe8f079d4f75ac57c93e4b8" @@ -709,16 +709,18 @@ version = "1.11.0" [[deps.StructUtils]] deps = ["Dates", "UUIDs"] -git-tree-sha1 = "9297459be9e338e546f5c4bedb59b3b5674da7f1" +git-tree-sha1 = "82bee338d650aa515f31866c460cb7e3bcef90b8" uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" -version = "2.6.2" +version = "2.8.2" [deps.StructUtils.extensions] StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsStaticArraysCoreExt = ["StaticArraysCore"] StructUtilsTablesExt = ["Tables"] [deps.StructUtils.weakdeps] Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [[deps.StyledStrings]] @@ -795,9 +797,9 @@ version = "1.11.0" [[deps.WeakRefStrings]] deps = ["DataAPI", "InlineStrings", "Parsers"] -git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +git-tree-sha1 = "0716e01c3b40413de5dedbc9c5c69f27cddfddfc" uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" -version = "1.4.2" +version = "1.4.3" [[deps.WorkerUtilities]] git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" @@ -826,6 +828,12 @@ git-tree-sha1 = "011b0a7331b41c25524b64dc42afc9683ee89026" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" version = "1.0.21+0" +[[deps.msghandler]] +deps = ["Arrow", "Base64", "DataFrames", "Dates", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"] +path = "." +uuid = "f2724d33-f338-4a57-b9f8-1be882570d10" +version = "0.5.6" + [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" diff --git a/Project.toml b/Project.toml index 364e5da..404b62a 100644 --- a/Project.toml +++ b/Project.toml @@ -18,4 +18,6 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] Base64 = "1.11.0" +Dates = "1.11.0" +GeneralUtils = "0.3.1" JSON = "1.4.0"