16 Commits

Author SHA1 Message Date
14cd4c0076 update doc 2026-03-24 08:55:06 +07:00
5191f1aae5 update 2026-03-23 16:15:15 +07:00
ton
c20a266e72 Merge pull request 'adopt_ASG_doc' (#12) from adopt_ASG_doc into v0.6.0-dev
Reviewed-on: #12
2026-03-23 08:00:20 +00:00
4f141b130e update 2026-03-23 14:50:00 +07:00
fa039f2820 update architecture.md 2026-03-23 13:33:26 +07:00
b0c5ecb942 update walkthrough doc 2026-03-23 13:03:27 +07:00
ba659368a5 update spec doc 2026-03-23 12:51:55 +07:00
1b38b3d6f1 update requirement doc 2026-03-23 11:52:38 +07:00
ton
ad28386ff0 Merge pull request 'remove_arrow_in_natsbridge_csr' (#11) from remove_arrow_in_natsbridge_csr into main
Reviewed-on: #11
2026-03-19 04:12:28 +00:00
9c4c941840 add js class 2026-03-15 18:41:45 +07:00
34d8e3fad8 update 2026-03-15 12:09:04 +07:00
49d7898720 remove arrow support for natsbridge_csr.js 2026-03-15 11:58:08 +07:00
fb315a0525 update 2026-03-14 11:24:27 +07:00
07acde45da update readme 2026-03-14 10:46:19 +07:00
3c6e139ac0 update readme 2026-03-14 10:41:16 +07:00
ton
50211b671d Merge pull request 'update_docs' (#10) from update_docs into main
Reviewed-on: #10
2026-03-14 00:53:02 +00:00
16 changed files with 2878 additions and 1668 deletions

View File

@@ -143,11 +143,29 @@ Since I develop src folder before I adopt SDD_FRAMEWORK.md approach, can you che
# ---------------------------------------------- 100 --------------------------------------------- # # ---------------------------------------------- 100 --------------------------------------------- #
Check NATSBridge/docs folder I want to update the content of the following files according to ASG_Framework/ASG_Framework.md:
- NATSBridge/docs/requirements.md
- NATSBridge/docs/specification.md
- NATSBridge/docs/ui-specification.md (you'll need to create this one)
- NATSBridge/docs/walkthrough.md
- NATSBridge/docs/architecture.md
I'll do the other docs not listed here later myself.
now help me update the following fileaccording to ASG_Framework/ASG_Framework.md:
- NATSBridge/docs/specification.md
<!-- ------------------------------------------- 100 ------------------------------------------- -->
Check NATSBridge/docs folder. I would like to expand this package to include Dart support.
Can you update the content of the following files according to ASG_Framework/ASG_Framework.md:
- NATSBridge/docs/requirements.md
- NATSBridge/docs/specification.md
- NATSBridge/docs/walkthrough.md
- NATSBridge/docs/architecture.md

View File

@@ -52,9 +52,9 @@ version = "1.2.2"
[[deps.CSV]] [[deps.CSV]]
deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"]
git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" git-tree-sha1 = "8d8e0b0f350b8e1c91420b5e64e5de774c2f0f4d"
uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
version = "0.10.15" version = "0.10.16"
[[deps.CodeTracking]] [[deps.CodeTracking]]
deps = ["InteractiveUtils", "UUIDs"] deps = ["InteractiveUtils", "UUIDs"]
@@ -108,9 +108,9 @@ version = "1.3.0+1"
[[deps.ConcurrentUtilities]] [[deps.ConcurrentUtilities]]
deps = ["Serialization", "Sockets"] deps = ["Serialization", "Sockets"]
git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8"
uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb"
version = "2.5.0" version = "2.5.1"
[[deps.Crayons]] [[deps.Crayons]]
git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
@@ -171,9 +171,9 @@ uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
version = "1.7.0" version = "1.7.0"
[[deps.EnumX]] [[deps.EnumX]]
git-tree-sha1 = "7bebc8aad6ee6217c78c5ddcf7ed289d65d0263e" git-tree-sha1 = "c49898e8438c828577f04b92fc9368c388ac783c"
uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
version = "1.0.6" version = "1.0.7"
[[deps.ExceptionUnwrapping]] [[deps.ExceptionUnwrapping]]
deps = ["Test"] deps = ["Test"]
@@ -234,9 +234,9 @@ version = "0.3.1"
[[deps.HTTP]] [[deps.HTTP]]
deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
git-tree-sha1 = "5e6fe50ae7f23d171f44e311c2960294aaa0beb5" git-tree-sha1 = "51059d23c8bb67911a2e6fd5130229113735fc7e"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "1.10.19" version = "1.11.0"
[[deps.HashArrayMappedTries]] [[deps.HashArrayMappedTries]]
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
@@ -307,9 +307,9 @@ weakdeps = ["ArrowTypes"]
[[deps.JuliaInterpreter]] [[deps.JuliaInterpreter]]
deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
git-tree-sha1 = "80580012d4ed5a3e8b18c7cd86cebe4b816d17a6" git-tree-sha1 = "3d3b79166e2a0afcf875df20db110af91ad3ab61"
uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
version = "0.10.9" version = "0.10.11"
[[deps.JuliaSyntaxHighlighting]] [[deps.JuliaSyntaxHighlighting]]
deps = ["StyledStrings"] deps = ["StyledStrings"]
@@ -383,9 +383,9 @@ version = "1.2.0"
[[deps.LoweredCodeUtils]] [[deps.LoweredCodeUtils]]
deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] deps = ["CodeTracking", "Compiler", "JuliaInterpreter"]
git-tree-sha1 = "65ae3db6ab0e5b1b5f217043c558d9d1d33cc88d" git-tree-sha1 = "5d4278f755440f70648d80cc6225f51e78e94094"
uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
version = "3.5.0" version = "3.5.1"
[[deps.Lz4_jll]] [[deps.Lz4_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"] deps = ["Artifacts", "JLLWrappers", "Libdl"]
@@ -400,9 +400,9 @@ version = "1.11.0"
[[deps.MbedTLS]] [[deps.MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"]
git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" git-tree-sha1 = "8785729fa736197687541f7053f6d8ab7fc44f92"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d" uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.1.9" version = "1.1.10"
[[deps.MbedTLS_jll]] [[deps.MbedTLS_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"] deps = ["Artifacts", "JLLWrappers", "Libdl"]
@@ -437,10 +437,10 @@ uuid = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
version = "0.1.0" version = "0.1.0"
[[deps.NATSBridge]] [[deps.NATSBridge]]
deps = ["Arrow", "DataFrames", "Dates", "GeneralUtils", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"] deps = ["Arrow", "Base64", "DataFrames", "Dates", "GeneralUtils", "HTTP", "JSON", "NATS", "PrettyPrinting", "Revise", "UUIDs"]
path = "." path = "."
uuid = "f2724d33-f338-4a57-b9f8-1be882570d10" uuid = "f2724d33-f338-4a57-b9f8-1be882570d10"
version = "0.4.1" version = "0.5.6"
[[deps.NanoDates]] [[deps.NanoDates]]
deps = ["Dates", "Parsers"] deps = ["Dates", "Parsers"]
@@ -514,9 +514,9 @@ version = "1.3.3"
[[deps.Preferences]] [[deps.Preferences]]
deps = ["TOML"] deps = ["TOML"]
git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4"
uuid = "21216c6a-2e73-6563-6e65-726566657250" uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.5.1" version = "1.5.2"
[[deps.PrettyPrinting]] [[deps.PrettyPrinting]]
git-tree-sha1 = "142ee93724a9c5d04d78df7006670a93ed1b244e" git-tree-sha1 = "142ee93724a9c5d04d78df7006670a93ed1b244e"
@@ -525,9 +525,15 @@ version = "0.4.2"
[[deps.PrettyTables]] [[deps.PrettyTables]]
deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"]
git-tree-sha1 = "c5a07210bd060d6a8491b0ccdee2fa0235fc00bf" git-tree-sha1 = "211530a7dc76ab59087f4d4d1fc3f086fbe87594"
uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
version = "3.1.2" version = "3.2.3"
[deps.PrettyTables.extensions]
PrettyTablesTypstryExt = "Typstry"
[deps.PrettyTables.weakdeps]
Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e"
[[deps.Printf]] [[deps.Printf]]
deps = ["Unicode"] deps = ["Unicode"]
@@ -535,9 +541,9 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
version = "1.11.0" version = "1.11.0"
[[deps.PtrArrays]] [[deps.PtrArrays]]
git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe"
uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d"
version = "1.3.0" version = "1.4.0"
[[deps.QuadGK]] [[deps.QuadGK]]
deps = ["DataStructures", "LinearAlgebra"] deps = ["DataStructures", "LinearAlgebra"]
@@ -568,9 +574,9 @@ version = "1.2.2"
[[deps.Revise]] [[deps.Revise]]
deps = ["CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Preferences", "REPL", "UUIDs"] deps = ["CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Preferences", "REPL", "UUIDs"]
git-tree-sha1 = "14d1bfb0a30317edc77e11094607ace3c800f193" git-tree-sha1 = "d97d78d4fc5f858d8ce44f6b88bc972f2023f51d"
uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe"
version = "3.13.2" version = "3.14.0"
[deps.Revise.extensions] [deps.Revise.extensions]
DistributedExt = "Distributed" DistributedExt = "Distributed"
@@ -596,9 +602,9 @@ version = "0.7.0"
[[deps.ScopedValues]] [[deps.ScopedValues]]
deps = ["HashArrayMappedTries", "Logging"] deps = ["HashArrayMappedTries", "Logging"]
git-tree-sha1 = "c3b2323466378a2ba15bea4b2f73b081e022f473" git-tree-sha1 = "ac4b837d89a58c848e85e698e2a2514e9d59d8f6"
uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" uuid = "7e506255-f358-4e82-b7e4-beb19740aa63"
version = "1.5.0" version = "1.6.0"
[[deps.Scratch]] [[deps.Scratch]]
deps = ["Dates"] deps = ["Dates"]
@@ -644,9 +650,9 @@ version = "1.12.0"
[[deps.SpecialFunctions]] [[deps.SpecialFunctions]]
deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
git-tree-sha1 = "f2685b435df2613e25fc10ad8c26dddb8640f547" git-tree-sha1 = "5acc6a41b3082920f79ca3c759acbcecf18a8d78"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b" uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "2.6.1" version = "2.7.1"
[deps.SpecialFunctions.extensions] [deps.SpecialFunctions.extensions]
SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" SpecialFunctionsChainRulesCoreExt = "ChainRulesCore"
@@ -692,9 +698,9 @@ version = "1.5.2"
[[deps.StringManipulation]] [[deps.StringManipulation]]
deps = ["PrecompileTools"] deps = ["PrecompileTools"]
git-tree-sha1 = "a3c1536470bf8c5e02096ad4853606d7c8f62721" git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e"
version = "0.4.2" version = "0.4.4"
[[deps.StringViews]] [[deps.StringViews]]
git-tree-sha1 = "f2dcb92855b31ad92fe8f079d4f75ac57c93e4b8" git-tree-sha1 = "f2dcb92855b31ad92fe8f079d4f75ac57c93e4b8"
@@ -709,16 +715,18 @@ version = "1.11.0"
[[deps.StructUtils]] [[deps.StructUtils]]
deps = ["Dates", "UUIDs"] deps = ["Dates", "UUIDs"]
git-tree-sha1 = "9297459be9e338e546f5c4bedb59b3b5674da7f1" git-tree-sha1 = "fa95b3b097bcef5845c142ea2e085f1b2591e92c"
uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42"
version = "2.6.2" version = "2.7.1"
[deps.StructUtils.extensions] [deps.StructUtils.extensions]
StructUtilsMeasurementsExt = ["Measurements"] StructUtilsMeasurementsExt = ["Measurements"]
StructUtilsStaticArraysCoreExt = ["StaticArraysCore"]
StructUtilsTablesExt = ["Tables"] StructUtilsTablesExt = ["Tables"]
[deps.StructUtils.weakdeps] [deps.StructUtils.weakdeps]
Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7"
StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
[[deps.StyledStrings]] [[deps.StyledStrings]]

View File

@@ -1,6 +1,6 @@
name = "NATSBridge" name = "NATSBridge"
uuid = "f2724d33-f338-4a57-b9f8-1be882570d10" uuid = "f2724d33-f338-4a57-b9f8-1be882570d10"
version = "0.5.5" version = "0.5.6"
authors = ["narawat <narawat@gmail.com>"] authors = ["narawat <narawat@gmail.com>"]
[deps] [deps]

552
README.md
View File

@@ -1,6 +1,6 @@
# NATSBridge - Cross-Platform Bi-Directional Data Bridge # NATSBridge - Cross-Platform Bi-Directional Data Bridge
A high-performance, bi-directional data bridge for **Julia, JavaScript, Python, and MicroPython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads. A high-performance, bi-directional data bridge for **Julia**, **JavaScript**, **Python**, **Dart**, and **MicroPython** applications using NATS (Core & JetStream), implementing the Claim-Check pattern for large payloads.
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
[![NATS](https://img.shields.io/badge/NATS-Enabled-green.svg)](https://nats.io) [![NATS](https://img.shields.io/badge/NATS-Enabled-green.svg)](https://nats.io)
@@ -28,8 +28,8 @@ NATSBridge enables seamless communication across multiple platforms through NATS
| Transport | Payload Size | Method | | Transport | Payload Size | Method |
|-----------|--------------|--------| |-----------|--------------|--------|
| **Direct** | < 1MB | Sent directly via NATS (Base64 encoded) | | **Direct** | < 500KB | Sent directly via NATS (Base64 encoded) |
| **Link** | >= 1MB | Uploaded to HTTP file server, URL sent via NATS | | **Link** | ≥ 500KB | Uploaded to HTTP file server, URL sent via NATS |
### Use Cases ### Use Cases
@@ -45,35 +45,39 @@ NATSBridge enables seamless communication across multiple platforms through NATS
| Platform | Implementation | Features | | Platform | Implementation | Features |
|----------|----------------|----------| |----------|----------------|----------|
| **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch | | **Julia** | [`src/NATSBridge.jl`](src/NATSBridge.jl) | Full feature set, Arrow IPC, multiple dispatch |
| **JavaScript** | [`src/natsbridge.js`](src/natsbridge.js) | Node.js, async/await | | **JavaScript (Node.js)** | [`src/natsbridge_ssr.js`](src/natsbridge_ssr.js) | Node.js, async/await, Arrow IPC |
| **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await | | **JavaScript (Browser)** | [`src/natsbridge_csr.js`](src/natsbridge_csr.js) | Browser, WebSocket NATS, async/await, JSON table only |
| **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints | | **Python** | [`src/natsbridge.py`](src/natsbridge.py) | Desktop Python, asyncio, type hints, Arrow IPC |
| **Dart (Desktop/Flutter)** | [`src/natsbridge.dart`](src/natsbridge.dart) | Desktop/Flutter, async/await, Arrow IPC |
| **Dart Web** | [`src/natsbridge.dart`](src/natsbridge.dart) | Web, WebSocket NATS, JSON table only |
| **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API | | **MicroPython** | [`src/natsbridge_mpy.py`](src/natsbridge_mpy.py) | Memory-constrained, synchronous API |
### Platform Comparison ### Platform Comparison
| Feature | Julia | JavaScript | JavaScript (Browser) | Python | MicroPython | | Feature | Julia | JavaScript | JavaScript (Browser) | Python | Dart | Dart Web | MicroPython |
|---------|-------|------------|----------------------|--------|-------------| |---------|-------|------------|----------------------|--------|------|----------|-------------|
| Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ | | Multiple Dispatch | ✅ Native | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) | | Async/Await | ❌ | ✅ Native | ✅ Native | ✅ Native | ✅ Native | ✅ Native | ⚠️ (uasyncio) |
| Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ❌ | | Type Safety | ✅ Strong | ⚠️ (TypeScript) | ⚠️ (TypeScript) | ✅ (Type hints) | ✅ Strong | ✅ Strong | ❌ |
| Arrow IPC | ✅ Native | ✅ | ✅ | ✅ | ❌ | | Arrow IPC | ✅ Native | ✅ Native | ❌ (Browser incompatible) | ✅ Native | ✅ Native | ❌ (Browser incompatible) | ❌ |
| Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ | | JSON Table | ✅ | ✅ | ✅ (Only table type) | ✅ | ✅ | ✅ | ⚠️ (Limited) |
| Link Transport | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) | | Direct Transport | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ | | Link Transport | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ⚠️ (Limited) |
| Cross-Platform API | ✅ | ✅ | ✅ | ✅ | ✅ | | Handler Functions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| WebSocket NATS | | | ✅ | | | | Cross-Platform API | | | ✅ | | ✅ | ✅ | ✅ |
| WebSocket NATS | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ |
--- ---
## Features ## Features
-**Cross-platform messaging** for Julia, JavaScript, Python, and MicroPython applications -**Cross-platform messaging** for Julia, JavaScript, Python, Dart, and MicroPython applications
-**Bi-directional messaging** with request-reply patterns -**Bi-directional messaging** with request-reply patterns
-**Multi-payload support** - send multiple payloads with different types in one message -**Multi-payload support** - send multiple payloads with different types in one message
-**Automatic transport selection** - direct vs link based on payload size -**Automatic transport selection** - direct vs link based on payload size
-**Claim-Check pattern** for payloads > 1MB -**Claim-Check pattern** for payloads ≥ 500KB
-**Apache Arrow IPC** support for tabular data (zero-copy reading) -**Apache Arrow IPC** support for tabular data (Desktop: Julia/Python/Node.js/Dart)
-**JSON Table** support for tabular data (All platforms including Browser)
-**Exponential backoff** for reliable file server downloads -**Exponential backoff** for reliable file server downloads
-**Correlation ID tracking** for message tracing -**Correlation ID tracking** for message tracing
-**Reply-to support** for request-response patterns -**Reply-to support** for request-response patterns
@@ -83,23 +87,24 @@ NATSBridge enables seamless communication across multiple platforms through NATS
## Quick Start ## Quick Start
### Step 1: Start NATS Server ### Prerequisites
```bash 1. **NATS Server** - Install and run a NATS server:
docker run -p 4222:4222 nats:latest ```bash
``` docker run -p 4222:4222 nats:latest
```
### Step 2: Start HTTP File Server (Optional) 2. **HTTP File Server** (optional, for large payloads) - Install and run a file server:
```bash
# Using Plik
docker run -p 8080:8080 -v /tmp/fileserver:/var/lib/plik -e PLIK_ADMIN_PASSWORD=admin plik/plik
# OR using simple Python HTTP server
mkdir -p /tmp/fileserver
python3 -m http.server 8080 --directory /tmp/fileserver
```
```bash ### Send Your First Message
# Create a directory for file uploads
mkdir -p /tmp/fileserver
# Start HTTP file server
python3 -m http.server 8080 --directory /tmp/fileserver
```
### Step 3: Send Your First Message
#### Julia #### Julia
@@ -107,14 +112,14 @@ python3 -m http.server 8080 --directory /tmp/fileserver
using NATSBridge using NATSBridge
data = [("message", "Hello World", "text")] data = [("message", "Hello World", "text")]
env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222") env, env_json_str = smartsend("/chat/room1", data; broker_url="nats://localhost:4222")
println("Message sent!") println("Message sent!")
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
const NATSBridge = require('./src/natsbridge.js'); import NATSBridge from './src/natsbridge_ssr.js';
const data = [["message", "Hello World", "text"]]; const data = [["message", "Hello World", "text"]];
const [env, env_json_str] = await NATSBridge.smartsend( const [env, env_json_str] = await NATSBridge.smartsend(
@@ -125,6 +130,20 @@ const [env, env_json_str] = await NATSBridge.smartsend(
console.log("Message sent!"); console.log("Message sent!");
``` ```
#### JavaScript (Browser)
```javascript
import NATSBridge from './src/natsbridge_csr.js';
const data = [["message", "Hello World", "text"]];
const [env, env_json_str] = await NATSBridge.smartsend(
"/chat/room1",
data,
{ broker_url: "ws://localhost:4222" }
);
console.log("Message sent!");
```
#### Python #### Python
```python ```python
@@ -139,6 +158,53 @@ env, env_json_str = await smartsend(
print("Message sent!") print("Message sent!")
``` ```
#### MicroPython
```python
from natsbridge import smartsend
data = [("message", "Hello World", "text")]
env, env_json_str = smartsend(
"/chat/room1",
data,
broker_url="nats://localhost:4222",
size_threshold=100000 # 100KB for MicroPython
)
print("Message sent!")
```
#### Dart (Desktop/Flutter)
```dart
import 'package:natsbridge/natsbridge.dart';
final data = [
['message', 'Hello World', 'text']
];
final [env, envJsonStr] = await NATSBridge.send(
'/chat/room1',
data,
brokerUrl: 'nats://localhost:4222',
);
print('Message sent!');
```
#### Dart Web
```dart
import 'package:natsbridge/natsbridge.dart';
final data = [
['message', 'Hello World', 'text']
];
final [env, envJsonStr] = await NATSBridge.send(
'/chat/room1',
data,
brokerUrl: 'ws://localhost:4222', // WebSocket for browser
);
print('Message sent!');
```
--- ---
## API Reference ## API Reference
@@ -147,13 +213,13 @@ print("Message sent!")
All platforms use the same input/output format for payloads: All platforms use the same input/output format for payloads:
**Input format for smartsend:** **Input format for `smartsend`:**
``` ```
[(dataname1, data1, type1), (dataname2, data2, type2), ...] [(dataname1, data1, type1), (dataname2, data2, type2), ...]
``` ```
**Output format for smartreceive:** **Output format for `smartreceive`:**
``` ```json
{ {
"correlation_id": "...", "correlation_id": "...",
"msg_id": "...", "msg_id": "...",
@@ -187,7 +253,7 @@ env, env_json_str = NATSBridge.smartsend(
broker_url::String = "nats://localhost:4222", broker_url::String = "nats://localhost:4222",
fileserver_url = "http://localhost:8080", fileserver_url = "http://localhost:8080",
fileserver_upload_handler::Function = plik_oneshot_upload, fileserver_upload_handler::Function = plik_oneshot_upload,
size_threshold::Int = 1_000_000, size_threshold::Int = 500_000,
correlation_id::String = string(uuid4()), correlation_id::String = string(uuid4()),
msg_purpose::String = "chat", msg_purpose::String = "chat",
sender_name::String = "NATSBridge", sender_name::String = "NATSBridge",
@@ -203,10 +269,10 @@ env, env_json_str = NATSBridge.smartsend(
# Returns: ::Tuple{msg_envelope_v1, String} # Returns: ::Tuple{msg_envelope_v1, String}
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
const NATSBridge = require('natsbridge'); import NATSBridge from './src/natsbridge_ssr.js';
const [env, env_json_str] = await NATSBridge.smartsend( const [env, env_json_str] = await NATSBridge.smartsend(
subject, subject,
@@ -215,7 +281,36 @@ const [env, env_json_str] = await NATSBridge.smartsend(
broker_url: 'nats://localhost:4222', broker_url: 'nats://localhost:4222',
fileserver_url: 'http://localhost:8080', fileserver_url: 'http://localhost:8080',
fileserver_upload_handler: NATSBridge.plikOneshotUpload, fileserver_upload_handler: NATSBridge.plikOneshotUpload,
size_threshold: 1_000_000, size_threshold: 500_000,
correlation_id: uuidv4(),
msg_purpose: 'chat',
sender_name: 'NATSBridge',
receiver_name: '',
receiver_id: '',
reply_to: '',
reply_to_msg_id: '',
is_publish: true,
nats_connection: null,
msg_id: uuidv4(),
sender_id: uuidv4()
}
);
// Returns: Promise<[env, env_json_str]>
```
#### JavaScript (Browser)
```javascript
import NATSBridge from './src/natsbridge_csr.js';
const [env, env_json_str] = await NATSBridge.smartsend(
subject,
data,
{
broker_url: 'ws://localhost:4222',
fileserver_url: 'http://localhost:8080',
fileserver_upload_handler: NATSBridge.plikOneshotUpload,
size_threshold: 500_000,
correlation_id: uuidv4(), correlation_id: uuidv4(),
msg_purpose: 'chat', msg_purpose: 'chat',
sender_name: 'NATSBridge', sender_name: 'NATSBridge',
@@ -243,7 +338,7 @@ env, env_json_str = await NATSBridge.smartsend(
broker_url: str = "nats://localhost:4222", broker_url: str = "nats://localhost:4222",
fileserver_url: str = "http://localhost:8080", fileserver_url: str = "http://localhost:8080",
fileserver_upload_handler: Callable = plik_oneshot_upload, fileserver_upload_handler: Callable = plik_oneshot_upload,
size_threshold: int = 1_000_000, size_threshold: int = 500_000,
correlation_id: str = None, correlation_id: str = None,
msg_purpose: str = "chat", msg_purpose: str = "chat",
sender_name: str = "NATSBridge", sender_name: str = "NATSBridge",
@@ -274,6 +369,54 @@ env, env_json_str = NATSBridge.smartsend(
# Returns: Tuple[Dict, str] # Returns: Tuple[Dict, str]
``` ```
#### Dart (Desktop/Flutter)
```dart
import 'package:natsbridge/natsbridge.dart';
final env, envJsonStr = await NATSBridge.send(
subject,
data, // List of [dataname, data, type] lists
brokerUrl: 'nats://localhost:4222',
fileserverUrl: 'http://localhost:8080',
fileserverUploadHandler: plikOneshotUpload,
sizeThreshold: 500000,
correlationId: uuid.v4(),
msgPurpose: 'chat',
senderName: 'NATSBridge',
receiverName: '',
receiverId: '',
replyTo: '',
replyToMsgId: '',
isPublish: true,
);
// Returns: Future<List<dynamic>> [env, env_json_str]
```
#### Dart Web
```dart
import 'package:natsbridge/natsbridge.dart';
final env, envJsonStr = await NATSBridge.send(
subject,
data, // List of [dataname, data, type] lists
brokerUrl: 'ws://localhost:4222', // WebSocket for browser
fileserverUrl: 'http://localhost:8080',
fileserverUploadHandler: plikOneshotUpload,
sizeThreshold: 500000,
correlationId: uuid.v4(),
msgPurpose: 'chat',
senderName: 'NATSBridge',
receiverName: '',
receiverId: '',
replyTo: '',
replyToMsgId: '',
isPublish: true,
);
// Returns: Future<List<dynamic>> [env, env_json_str]
```
### smartreceive ### smartreceive
Receives and processes messages from NATS, handling both direct and link transport. Receives and processes messages from NATS, handling both direct and link transport.
@@ -293,9 +436,28 @@ env = NATSBridge.smartreceive(
# Returns: ::JSON.Object{String, Any} # Returns: ::JSON.Object{String, Any}
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
import NATSBridge from './src/natsbridge_ssr.js';
const env = await NATSBridge.smartreceive(
msg,
{
fileserver_download_handler: NATSBridge.fetchWithBackoff,
max_retries: 5,
base_delay: 100,
max_delay: 5000
}
);
// Returns: Promise<env_object>
```
#### JavaScript (Browser)
```javascript
import NATSBridge from './src/natsbridge_csr.js';
const env = await NATSBridge.smartreceive( const env = await NATSBridge.smartreceive(
msg, msg,
{ {
@@ -311,6 +473,8 @@ const env = await NATSBridge.smartreceive(
#### Python #### Python
```python ```python
from natsbridge import NATSBridge
env = await NATSBridge.smartreceive( env = await NATSBridge.smartreceive(
msg, msg,
fileserver_download_handler=fetch_with_backoff, fileserver_download_handler=fetch_with_backoff,
@@ -324,6 +488,8 @@ env = await NATSBridge.smartreceive(
#### MicroPython #### MicroPython
```python ```python
from natsbridge import NATSBridge
env = NATSBridge.smartreceive( env = NATSBridge.smartreceive(
msg, msg,
fileserver_download_handler=_sync_fileserver_download, fileserver_download_handler=_sync_fileserver_download,
@@ -334,20 +500,50 @@ env = NATSBridge.smartreceive(
# Returns: Dict with "payloads" key # Returns: Dict with "payloads" key
``` ```
#### Dart (Desktop/Flutter)
```dart
import 'package:natsbridge/natsbridge.dart';
final env = await NATSBridge.receive(
msg,
fileserverDownloadHandler: fetchWithBackoff,
maxRetries: 5,
baseDelay: 100,
maxDelay: 5000,
);
// Returns: Future<Map<String, dynamic>> with "payloads" key
```
#### Dart Web
```dart
import 'package:natsbridge/natsbridge.dart';
final env = await NATSBridge.receive(
msg,
fileserverDownloadHandler: fetchWithBackoff,
maxRetries: 5,
baseDelay: 100,
maxDelay: 5000,
);
// Returns: Future<Map<String, dynamic>> with "payloads" key
```
--- ---
## Payload Types ## Payload Types
| Type | Julia | JavaScript | Python | MicroPython | Description | | Type | Julia | JavaScript | Python | Dart | Dart Web | MicroPython | Description |
|------|-------|------------|--------|-------------|-------------| |------|-------|------------|--------|------|----------|-------------|-------------|
| `text` | `String` | `string` | `str` | `str` | Plain text strings | | `text` | `String` | `string` | `str` | `String` | `String` | `str` | Plain text strings |
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable dictionaries | | `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `Map` | `Map` | `dict` | JSON-serializable dictionaries |
| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array<Object>` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | | `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | `List<Map>` (Desktop/Flutter) | ❌ (Browser incompatible) | ❌ | Tabular data (Arrow IPC) |
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ❌ | Tabular data (JSON) | | `jsontable` | `DataFrame`, `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | `List<Map>` | `List<Map>` | ⚠️ | Tabular data (JSON) - **Only table type in Browser/Dart Web** |
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image data (PNG, JPG) | | `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `Uint8List` | `Uint8List` | `bytearray` | Image data (PNG, JPG) |
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio data (WAV, MP3) | | `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `Uint8List` | `Uint8List` | `bytearray` | Audio data (WAV, MP3) |
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video data (MP4, AVI) | | `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `Uint8List` | `Uint8List` | `bytearray` | Video data (MP4, AVI) |
| `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | `bytearray` | Generic binary data | | `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | `Uint8List` | `Uint8List` | `bytearray` | Generic binary data |
--- ---
@@ -368,13 +564,13 @@ data = [
("large_document", large_file_data, "binary") ("large_document", large_file_data, "binary")
] ]
env, env_json_str = NATSBridge.smartsend("/chat/room1", data; fileserver_url="http://localhost:8080") env, env_json_str = smartsend("/chat/room1", data; fileserver_url="http://localhost:8080")
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
const NATSBridge = require('natsbridge'); import NATSBridge from './src/natsbridge_ssr.js';
const data = [ const data = [
["message_text", "Hello!", "text"], ["message_text", "Hello!", "text"],
@@ -389,6 +585,24 @@ const [env, env_json_str] = await NATSBridge.smartsend(
); );
``` ```
#### JavaScript (Browser)
```javascript
import NATSBridge from './src/natsbridge_csr.js';
const data = [
["message_text", "Hello!", "text"],
["user_avatar", imageData, "image"],
["large_document", largeFileData, "binary"]
];
const [env, env_json_str] = await NATSBridge.smartsend(
"/chat/room1",
data,
{ broker_url: 'ws://localhost:4222', fileserver_url: 'http://localhost:8080' }
);
```
#### Python #### Python
```python ```python
@@ -423,13 +637,13 @@ config = Dict(
) )
data = [("config", config, "dictionary")] data = [("config", config, "dictionary")]
env, env_json_str = NATSBridge.smartsend("/device/config", data) env, env_json_str = smartsend("/device/config", data)
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
const NATSBridge = require('natsbridge'); import NATSBridge from './src/natsbridge_ssr.js';
const config = { const config = {
wifi_ssid: "MyNetwork", wifi_ssid: "MyNetwork",
@@ -475,13 +689,13 @@ df = DataFrame(
) )
data = [("students", df, "arrowtable")] data = [("students", df, "arrowtable")]
env, env_json_str = NATSBridge.smartsend("/data/analysis", data) env, env_json_str = smartsend("/data/analysis", data)
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
const NATSBridge = require('natsbridge'); import NATSBridge from './src/natsbridge_ssr.js';
const df = [ const df = [
{ id: 1, name: "Alice", score: 95 }, { id: 1, name: "Alice", score: 95 },
@@ -511,6 +725,26 @@ data = [("students", df, "arrowtable")]
env, env_json_str = await NATSBridge.smartsend("/data/analysis", data) env, env_json_str = await NATSBridge.smartsend("/data/analysis", data)
``` ```
#### JavaScript (Browser)
```javascript
import NATSBridge from './src/natsbridge_csr.js';
// Browser uses jsontable (JSON array of objects) instead of arrowtable
// Apache Arrow is not compatible with browsers
const df = [
{ id: 1, name: "Alice", score: 95 },
{ id: 2, name: "Bob", score: 88 },
{ id: 3, name: "Charlie", score: 92 }
];
const [env, env_json_str] = await NATSBridge.smartsend(
"/data/analysis",
[["students", df, "jsontable"]], // Use jsontable for browser
{ broker_url: 'ws://localhost:4222' }
);
```
### Example 4: Request-Response Pattern ### Example 4: Request-Response Pattern
Bi-directional communication with reply-to support. Bi-directional communication with reply-to support.
@@ -521,18 +755,29 @@ Bi-directional communication with reply-to support.
using NATSBridge using NATSBridge
# Requester # Requester
env, env_json_str = NATSBridge.smartsend( env, env_json_str = smartsend(
"/device/command", "/device/command",
[("command", Dict("action" => "read_sensor"), "dictionary")]; [("command", Dict("action" => "read_sensor"), "dictionary")];
broker_url="nats://localhost:4222", broker_url="nats://localhost:4222",
reply_to="/device/response" reply_to="/device/response"
) )
# Receiver (in separate application)
msg = NATS.subscription.next()
env = smartreceive(msg)
# Process request and send response
response_env, response_json = smartsend(
"/device/response",
[("result", Dict("value" => 42), "dictionary")],
reply_to="/device/command",
reply_to_msg_id=env["msg_id"]
)
``` ```
#### JavaScript #### JavaScript (Node.js)
```javascript ```javascript
const NATSBridge = require('natsbridge'); import NATSBridge from './src/natsbridge_ssr.js';
// Requester // Requester
const [env, env_json_str] = await NATSBridge.smartsend( const [env, env_json_str] = await NATSBridge.smartsend(
@@ -540,6 +785,16 @@ const [env, env_json_str] = await NATSBridge.smartsend(
[["command", { action: "read_sensor" }, "dictionary"]], [["command", { action: "read_sensor" }, "dictionary"]],
{ broker_url: 'nats://localhost:4222', reply_to: '/device/response' } { broker_url: 'nats://localhost:4222', reply_to: '/device/response' }
); );
// Receiver (in separate application)
// const msg = await natsConsumer.next();
// const env = await NATSBridge.smartreceive(msg);
// Process request and send response
// const response_env, response_json = await NATSBridge.smartsend(
// "/device/response",
// [["result", { value: 42 }, "dictionary"]],
// { reply_to: '/device/command', reply_to_msg_id: env.msg_id }
// );
``` ```
#### Python #### Python
@@ -554,6 +809,17 @@ env, env_json_str = await NATSBridge.smartsend(
broker_url="nats://localhost:4222", broker_url="nats://localhost:4222",
reply_to="/device/response" reply_to="/device/response"
) )
# Receiver (in separate application)
# msg = await nats_consumer.next()
# env = await NATSBridge.smartreceive(msg)
# Process request and send response
# response_env, response_json = await NATSBridge.smartsend(
# "/device/response",
# [("result", {"value": 42}, "dictionary")],
# reply_to="/device/command",
# reply_to_msg_id=env["msg_id"]
# )
``` ```
--- ---
@@ -567,6 +833,7 @@ env, env_json_str = await NATSBridge.smartsend(
| **Julia** | `test/test_julia_*_sender.jl` | `test/test_julia_*_receiver.jl` | | **Julia** | `test/test_julia_*_sender.jl` | `test/test_julia_*_receiver.jl` |
| **JavaScript** | `test/test_js_*_sender.js` | `test/test_js_*_receiver.js` | | **JavaScript** | `test/test_js_*_sender.js` | `test/test_js_*_receiver.js` |
| **Python** | `test/test_py_*_sender.py` | `test/test_py_*_receiver.py` | | **Python** | `test/test_py_*_sender.py` | `test/test_py_*_receiver.py` |
| **Dart** | `test/test_dart_*_sender.dart` | `test/test_dart_*_receiver.dart` |
### Run Tests ### Run Tests
@@ -634,16 +901,157 @@ python3 test/test_py_table_sender.py
python3 test/test_py_table_receiver.py python3 test/test_py_table_receiver.py
``` ```
#### Dart
```bash
# Text message exchange
dart test/test_dart_text_sender.dart
dart test/test_dart_text_receiver.dart
# Dictionary exchange
dart test/test_dart_dictionary_sender.dart
dart test/test_dart_dictionary_receiver.dart
# Binary transfer
dart test/test_dart_binary_sender.dart
dart test/test_dart_binary_receiver.dart
# Mixed payload types
dart test/test_dart_mix_payloads_sender.dart
dart test/test_dart_mix_payloads_receiver.dart
# Table exchange
dart test/test_dart_table_sender.dart
dart test/test_dart_table_receiver.dart
```
---
## Browser Deployment
### Using with Node.js Build Tools
The browser implementation (`src/natsbridge_csr.js`) can be bundled for production deployment using modern JavaScript build tools.
#### Prerequisites
```bash
# Install the browser-compatible NATS client
npm install nats.ws
```
#### Vite (Recommended)
```bash
npm create vite@latest my-app -- --template vanilla
cd my-app
npm install nats.ws
```
In `vite.config.js`:
```javascript
import { defineConfig } from 'vite';
export default defineConfig({
resolve: {
alias: {
'nats.ws': 'nats.ws/dist/esm/browser.js'
}
}
});
```
Build command:
```bash
npm run build # Outputs to dist/ folder
```
#### Webpack
```bash
npm install webpack webpack-cli --save-dev
npm install nats.ws
```
In `webpack.config.js`:
```javascript
module.exports = {
entry: './src/index.js',
output: {
filename: 'bundle.js',
path: __dirname + '/dist'
},
resolve: {
alias: {
'nats.ws': 'nats.ws/dist/esm/browser.js'
}
}
};
```
Build command:
```bash
npx webpack
```
#### esbuild (Simple & Fast)
```bash
npm install esbuild nats.ws --save-dev
```
Create `build.js`:
```javascript
import esbuild from 'esbuild';
esbuild.buildSync({
entryPoints: ['src/natsbridge_csr.js'],
bundle: true,
outfile: 'dist/natsbridge-csr-bundle.js',
format: 'esm',
platform: 'browser',
target: 'es2020'
});
```
Build command:
```bash
node build.js
```
### Using in Your HTML
```html
<!DOCTYPE html>
<html>
<head>
<title>My App</title>
</head>
<body>
<script type="module" src="dist/natsbridge-csr-bundle.js"></script>
<script type="module">
import NATSBridgeCSR from './dist/natsbridge-csr-bundle.js';
// Use the library
const [env, envJson] = await NATSBridgeCSR.smartsend(
"/chat/user/v1/message",
[["msg", "Hello", "text"]],
{ broker_url: "wss://nats.example.com" }
);
</script>
</body>
</html>
```
--- ---
## Documentation ## Documentation
For detailed architecture and implementation information, see: For detailed architecture and implementation information, see:
- [Architecture Documentation](docs/architecture_updated.md) - Cross-platform architecture, API parity, platform-specific patterns - [`docs/architecture.md`](docs/architecture.md) - Cross-platform architecture, API parity, platform-specific patterns
- [Implementation Guide](docs/implementation_updated.md) - Detailed implementation for each platform, handler functions, testing - [`docs/requirements.md`](docs/requirements.md) - Business requirements and user stories
- [Tutorial](docs/tutorial_updated.md) - Step-by-step getting started guide - [`docs/spec.md`](docs/spec.md) - Technical specification and contracts
- [Walkthrough](docs/walkthrough_updated.md) - Real-world application building guides - [`docs/walkthrough.md`](docs/walkthrough.md) - Real-world application building guides
--- ---

View File

@@ -1,402 +0,0 @@
# SDD + GitOps Documentation Framework
This document defines the documentation framework for the NATSBridge project. It establishes a structured approach to creating, maintaining, and evolving technical documentation in alignment with GitOps principles—ensuring that documentation is versioned, auditable, and continuously validated alongside the codebase.
---
## The SDD Framework: Seven Pillars of Documentation
| Document | Purpose (Rationale) | Primary Audience | Format / Content | Example (SaaS Context) | Measurement (KPI) |
|----------|---------------------|-----------------|------------------|------------------------|-------------------|
| **Requirements** | Capture the **business intent** — why we're building this and what success looks like. Defines boundaries and user-visible outcomes. | Stakeholders, Product Owners, Lead Developers | User stories, PRDs, acceptance criteria, non-functional constraints. | "System must process tabular data from Julia to SvelteKit UI with <200ms latency for 5-member teams." | 95% of requests complete <200ms (synthetic monitoring). |
| **Specification** | The **technical contract** — precise rules for inputs, outputs, and data shape. Ensures consistency across dev and test. | Developers, QA Engineers, CI/CD pipelines | OpenAPI, Protobuf, AsyncAPI. Endpoint definitions, schemas, error codes. | `contract.yaml` defining a NATS subject that accepts Arrow streams with snake_case headers. | 100% of messages validated against spec (CI block rate). |
| **Architecture** | The **blueprint** — how components fit together, interact, and scale. Guides system structure and trade-offs. | Architects, Senior Developers, DevOps | C4 diagrams, Mermaid.js, component/network/storage models. | Diagram showing 6-node cluster routing traffic via Caddy → Node.js API → Julia pods. | 100% of major decisions logged with trade-off analysis. |
| **Walkthrough** | The **story of flow** — shows how pieces connect end-to-end and why steps are sequenced. Builds intuition for new devs. | New Developers, Team Members | TOUR.md, Loom videos, sequence diagrams. Step-by-step traces with rationale. | "UI sends JSON → Node.js wraps Claim-Check → Julia pulls Arrow data (prevents NATS overflow)." | New developers ship feature in <2 days (PR timeline). |
| **Implementation** | The **real code** — business logic, helpers, tests, configs. Where design becomes executable. | Developers, Code Reviewers | Source code, README.md, unit tests, setup scripts. | Julia function for matrix calculation + SvelteKit component rendering table. | >80% unit test coverage, <5% drift from spec. |
| **Validation** | The **enforcer** — ensures implementation matches the spec. Blocks drift and human error. | Automation servers, QA, Lead Developers | CI jobs, contract tests, linting, integration checks. | CI job rejects PR with camelCase field not allowed by YAML spec. | <1% of PRs bypass validation gates. |
| **Runbook** | The **operational manual** — how the system lives in production, scales, and recovers. Guides on-call engineers. | DevOps, SREs, On-call Developers | K8s manifests, Helm charts, Markdown guides. Deployment, scaling, backup/restore, troubleshooting. | GitOps manifest ensuring 6 Julia replicas restart if memory >80%. | MTTR <15 minutes for P1 incidents. |
---
## Detailed Document Descriptions
### 1. Requirements
**Purpose**: Capture the *business intent* — why we're building this and what success looks like. Defines boundaries and user-visible outcomes.
**Why It Matters**:
- Aligns engineering efforts with business goals
- Provides a north star for feature development
- Establishes acceptance criteria before implementation begins
- Creates a contract between product and engineering
**Content Guidelines**:
- User stories with clear acceptance criteria (As a X, I want Y so that Z)
- Product Requirements Documents (PRDs) with success metrics
- Non-functional requirements (performance, security, scalability)
- Boundary definitions (what's in scope vs. out of scope)
**Best Practices**:
- Link each requirement to a measurable KPI
- Keep requirements testable and verifiable
- Maintain backward compatibility with existing requirements
- Review and update requirements as business context changes
---
### 2. Specification
**Purpose**: The *technical contract* — precise rules for inputs, outputs, and data shape. Ensures consistency across dev and test.
**Why It Matters**:
- Prevents implementation drift between components
- Enables contract testing in CI/CD pipelines
- Provides a single source of truth for data structures
- Facilitates integration between teams
**Content Guidelines**:
- API endpoint definitions (methods, paths, parameters)
- Request/response schemas (JSON, XML, Protobuf, AsyncAPI)
- Error codes and their meanings
- Data validation rules and constraints
- Rate limiting and quota definitions
**Best Practices**:
- Use formal specification languages (OpenAPI 3.0+, AsyncAPI)
- Version specifications alongside code
- Generate client SDKs from specifications
- Block CI on specification violations
- Document edge cases and error scenarios
---
### 3. Architecture
**Purpose**: The *blueprint* — how components fit together, interact, and scale. Guides system structure and trade-offs.
**Why It Matters**:
- Provides a mental model for system design
- Guides technical decision-making and trade-off analysis
- Facilitates onboarding of new architects and senior developers
- Documents scaling and performance considerations
**Content Guidelines**:
- C4 diagrams (Context, Container, Component levels)
- Mermaid.js flowcharts for sequence diagrams
- Component interaction diagrams
- Network topology and data flow
- Storage and caching strategies
- Scaling and resilience patterns
**Best Practices**:
- Use diagrams that are easy to update (Mermaid.js over static images)
- Document trade-off decisions with Rationale Documents
- Include scaling considerations for each component
- Document failure modes and recovery strategies
- Keep architecture diagrams versioned with code
---
### 4. Walkthrough
**Purpose**: The *story of flow* — shows how pieces connect end-to-end and why steps are sequenced. Builds intuition for new devs.
**Why It Matters**:
- Reduces onboarding time for new developers
- Provides context that code comments alone cannot convey
- Explains the "why" behind architectural decisions
- Helps identify gaps in the system design
**Content Guidelines**:
- Step-by-step flow descriptions with rationale
- Sequence diagrams showing request/response patterns
- "Tour of the codebase" guides
- Video walkthroughs (Loom, internal recordings)
- Debugging and tracing examples
**Best Practices**:
- Walk through real user journeys, not just technical flows
- Include "what could go wrong" scenarios
- Link walkthroughs to relevant code locations
- Keep walkthroughs updated with architecture changes
- Make walkthroughs interactive where possible
---
### 5. Implementation
**Purpose**: The *real code* — business logic, helpers, tests, configs. Where design becomes executable.
**Why It Matters**:
- This is the actual artifact that runs in production
- Code is the ultimate source of truth (when it matches spec)
- Tests validate correctness and prevent regressions
- Configuration files define runtime behavior
**Content Guidelines**:
- Business logic implementation
- Helper functions and utilities
- Unit and integration tests
- Configuration files (YAML, JSON, environment)
- Setup and development scripts
- Code organization and module structure
**Best Practices**:
- Follow consistent code style and conventions
- Write tests before or alongside implementation (TDD/BDD)
- Document complex logic with inline comments
- Keep configuration externalized and versioned
- Use type annotations where applicable
---
### 6. Validation
**Purpose**: The *enforcer* — ensures implementation matches the spec. Blocks drift and human error.
**Why It Matters**:
- Prevents breaking changes from reaching production
- Catches specification violations early in the CI pipeline
- Maintains data integrity and API consistency
- Reduces manual QA effort through automation
**Content Guidelines**:
- CI/CD pipeline configurations
- Contract testing scripts
- Linting rules and configurations
- Integration test suites
- Schema validation jobs
- Security scanning and audit jobs
**Best Practices**:
- Fail CI on specification violations
- Run validation jobs on every commit and PR
- Use automated code review tools
- Maintain validation job health dashboard
- Document validation failure remediation steps
---
### 7. Runbook
**Purpose**: The *operational manual* — how the system lives in production, scales, and recovers. Guides on-call engineers.
**Why It Matters**:
- Reduces Mean Time To Recovery (MTTR) for incidents
- Provides step-by-step guidance for common issues
- Documents scaling and deployment procedures
- Ensures operational knowledge is not siloed
**Content Guidelines**:
- Deployment procedures (manual and automated)
- Scaling instructions (horizontal/vertical)
- Backup and restore procedures
- Troubleshooting guides for common issues
- Runbook entries for specific error codes
- Contact information and escalation paths
**Best Practices**:
- Write runbooks for every P1/P2 incident
- Include exact commands and configuration snippets
- Test runbooks periodically (chaos engineering)
- Link runbook entries to relevant documentation
- Keep runbooks updated when system changes
---
## How to Use This Approach Effectively
### 1. Start with Requirements
Before writing any code or documentation, establish clear requirements. Ask:
- What business problem are we solving?
- How will we measure success?
- What are the non-negotiable constraints?
**Action**: Create a `docs/requirements/` directory and start with `PRD.md` and `KPIs.md`.
### 2. Define the Specification First
Once requirements are stable, define the technical specification. This becomes the contract for implementation.
**Action**: Create `docs/specification/` with `contract.yaml` (or appropriate format) and `error-codes.md`.
### 3. Design the Architecture
With requirements and specification in place, design the architecture. Document trade-off decisions explicitly.
**Action**: Create `docs/architecture/` with Mermaid diagrams and `trade-offs.md`.
### 4. Create Walkthroughs Early
As soon as the architecture is defined, create walkthroughs. This helps identify gaps and provides onboarding material.
**Action**: Create `docs/walkthrough/` with `TOUR.md` and sequence diagrams.
### 5. Implement with Validation in Mind
Write implementation code that adheres to the specification. Build validation into the CI pipeline from day one.
**Action**: Ensure test files are co-located with implementation and run on every commit.
### 6. Automate Validation
Build automated validation that runs in CI/CD. This ensures spec compliance and prevents drift.
**Action**: Configure CI jobs to validate against specification and block PRs on violations.
### 7. Document Operations from Day One
Create runbook entries as soon as deployment procedures are established. Update them when incidents occur.
**Action**: Create `docs/runbook/` with entries for deployment, scaling, and common issues.
---
## GitOps Integration
This documentation framework aligns with GitOps principles:
| GitOps Principle | Documentation Alignment |
|-----------------|------------------------|
| **Versioned** | All documentation lives in git, with history and audit trail |
| ** declarative** | Specifications and architecture are declarative contracts |
| **Automated** | Validation jobs automate spec compliance checks |
| **Self-Service** | Walkthroughs and runbooks enable self-service onboarding and operations |
| **Observability** | KPIs and metrics are defined for each documentation artifact |
**Git Structure**:
```
docs/
├── requirements/ # PRDs, user stories, KPIs
├── specification/ # OpenAPI, Protobuf, AsyncAPI specs
├── architecture/ # C4 diagrams, Mermaid, trade-off docs
├── walkthrough/ # TOUR.md, sequence diagrams
├── implementation/ # Source code (in src/)
├── validation/ # CI configs, test suites
└── runbook/ # Deployment, scaling, troubleshooting
```
---
## Metrics and Continuous Improvement
Each documentation artifact has associated KPIs. Track these to ensure quality:
| Document | KPI | Target |
|----------|-----|--------|
| Requirements | Requirement coverage | 100% of features have associated requirements |
| Specification | Spec compliance rate | 100% of messages validate against spec |
| Architecture | Decision documentation | 100% of major decisions logged with trade-offs |
| Walkthrough | New dev time-to-first-PR | <2 days from onboarding to first contribution |
| Implementation | Test coverage | >80% unit test coverage |
| Validation | Bypass rate | <1% of PRs bypass validation gates |
| Runbook | MTTR | <15 minutes for P1 incidents |
**Review Cadence**:
- Weekly: Review KPI dashboards and documentation gaps
- Monthly: Update documentation based on incident learnings
- Quarterly: Full framework review and improvement
---
## Template Examples
### Requirements Template
```markdown
# PRD: Feature Name
## Business Goal
[What problem are we solving?]
## Success Metrics
- [Metric 1]: Target [value]
- [Metric 2]: Target [value]
## User Stories
- As a [role], I want [feature] so that [benefit]
- Acceptance Criteria: [details]
## Non-Functional Requirements
- Performance: [details]
- Security: [details]
- Scalability: [details]
## Out of Scope
- [What's explicitly excluded]
```
### Specification Template
```yaml
# contract.yaml
openapi: 3.0.0
info:
title: NATSBridge API
version: 1.0.0
paths:
/api/v1/endpoint:
post:
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/Request'
responses:
'200':
description: Success
content:
application/json:
schema:
$ref: '#/components/schemas/Response'
```
### Architecture Template
```mermaid
%%{init: {'theme': 'base', 'themeVariables': {'primaryColor': '#3b82f6'}}}%%
flowchart TD
A[Client] --> B[Caddy]
B --> C[Node.js API]
C --> D[Julia Worker]
D --> E[NATS Cluster]
E --> F[Storage]
style A fill:#f9f9f9,stroke:#333
style E fill:#e0e7ff,stroke:#3b82f6
```
### Runbook Template
```markdown
# Runbook: Service Restart
**Severity**: P2
**Estimated Time**: 5 minutes
## Symptoms
- Service is unresponsive
- Health checks are failing
## Steps
1. SSH to the host
2. Run: `kubectl rollout restart deployment/natsbridge`
3. Monitor: `kubectl get pods -l app=natsbridge -w`
## Rollback
- Run: `kubectl rollout undo deployment/natsbridge`
## Post-Incident
- [ ] Review logs for root cause
- [ ] Update runbook if needed
```
---
## Conclusion
This SDD + GitOps Documentation Framework ensures that documentation is:
- **Structured**: Seven distinct artifacts with clear purposes
- **Automated**: Validation and CI/CD integration
- **Versioned**: All documentation in git with history
- **Measurable**: KPIs for quality and effectiveness
- **Actionable**: Practical templates and examples
Use this framework as a living document—update it as your team's needs evolve.

View File

@@ -1,16 +1,16 @@
# Architecture Documentation: NATSBridge # Architecture Documentation: NATSBridge
**Version**: 1.0.0 **Version**: 1.1.0
**Date**: 2026-03-13 **Date**: 2026-03-23
**Status**: Active **Status**: Active
**Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl) **Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl)
**Architecture Level**: C4 Container Level **Architecture Level**: C4 Container Level
--- ---
## Executive Summary ## 1. Executive Summary
This document defines the **blueprint** for NATSBridge - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, and **MicroPython** applications using NATS as the message bus. This document defines the **blueprint** for NATSBridge - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, and **MicroPython** applications using NATS as the message bus.
This architecture document serves as the single source of truth for: This architecture document serves as the single source of truth for:
- **System Structure**: How components fit together and interact - **System Structure**: How components fit together and interact
@@ -18,8 +18,29 @@ This architecture document serves as the single source of truth for:
- **Failure Modes**: How the system handles failures and recovers - **Failure Modes**: How the system handles failures and recovers
- **Trade-off Decisions**: The rationale behind architectural decisions - **Trade-off Decisions**: The rationale behind architectural decisions
### 1.1 Specification Traceability
| Architecture Section | Specification Reference | UI Specification Reference | Requirement ID(s) |
|---------------------|-------------------------|---------------------------|-------------------|
| Section 2 (Context Diagram) | specification.md:2 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
| Section 3 (Container Diagram) | specification.md:2, specification.md:3, specification.md:11 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
| Section 4 (Component Diagram) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
| Section 5 (High-Level) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
| Section 6 (Message Envelope) | specification.md:2, specification.md:3, specification.md:8 | - | FR-011, FR-012, FR-013, FR-014, NFR-401, NFR-403 |
| Section 7 (Payload Type) | specification.md:3, specification.md:5, specification.md:6 | - | FR-001, FR-002, FR-003, FR-006, FR-012, NFR-101, NFR-102 |
| Section 8 (Transport Strategy) | specification.md:6, specification.md:7 | - | FR-003, FR-004, FR-005, FR-010, NFR-104, NFR-105, NFR-106 |
| Section 9 (Platform-Specific) | specification.md:13, specification.md:14 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
| Section 10 (Scaling) | specification.md:7, specification.md:13 | - | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 |
| Section 11 (Failure Modes) | specification.md:9, specification.md:11 | - | FR-008, FR-009, FR-010, FR-011, NFR-201, NFR-202, NFR-203 |
| Section 12 (Trade-offs) | specification.md:2, specification.md:3, specification.md:6, specification.md:7 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 |
| Section 13 (Deployment) | specification.md:12, specification.md:18 | - | FR-013, FR-014, NFR-201, NFR-203 |
| Section 14 (Security) | specification.md:4, specification.md:9, specification.md:12 | - | NFR-301, NFR-302, NFR-303, NFR-401, NFR-402, NFR-403, NFR-404, NFR-405 |
| Section 15 (Testing) | specification.md:17 | - | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
--- ---
## 2. Architecture Overview
## Architecture Overview ## Architecture Overview
### C4 Context Diagram ### C4 Context Diagram
@@ -35,17 +56,20 @@ flowchart TD
Julia_App[Julia Application] Julia_App[Julia Application]
JS_App[JavaScript Application<br/>Node.js/Browser] JS_App[JavaScript Application<br/>Node.js/Browser]
Python_App[Python Application<br/>Desktop] Python_App[Python Application<br/>Desktop]
Dart_App[Dart Application<br/>Desktop/Flutter/Web]
MicroPython_App[MicroPython Device] MicroPython_App[MicroPython Device]
end end
Julia_App -->|NATS| NATS_Server Julia_App -->|NATS| NATS_Server
JS_App -->|NATS| NATS_Server JS_App -->|NATS| NATS_Server
Python_App -->|NATS| NATS_Server Python_App -->|NATS| NATS_Server
Dart_App -->|NATS| NATS_Server
MicroPython_App -->|NATS| NATS_Server MicroPython_App -->|NATS| NATS_Server
Julia_App -->|HTTP| File_Server Julia_App -->|HTTP| File_Server
JS_App -->|HTTP| File_Server JS_App -->|HTTP| File_Server
Python_App -->|HTTP| File_Server Python_App -->|HTTP| File_Server
Dart_App -->|HTTP| File_Server
MicroPython_App -->|HTTP| File_Server MicroPython_App -->|HTTP| File_Server
style NATS_Server fill:#fff3e0,stroke:#f57c00 style NATS_Server fill:#fff3e0,stroke:#f57c00
@@ -53,6 +77,7 @@ flowchart TD
style Julia_App fill:#e8f5e9,stroke:#4caf50 style Julia_App fill:#e8f5e9,stroke:#4caf50
style JS_App fill:#e3f2fd,stroke:#2196f3 style JS_App fill:#e3f2fd,stroke:#2196f3
style Python_App fill:#e3f2fd,stroke:#2196f3 style Python_App fill:#e3f2fd,stroke:#2196f3
style Dart_App fill:#fff0f6,stroke:#e91e63
style MicroPython_App fill:#fce4ec,stroke:#e91e63 style MicroPython_App fill:#fce4ec,stroke:#e91e63
``` ```
@@ -64,6 +89,7 @@ flowchart TD
Julia_Module[Julia NATSBridge Module] Julia_Module[Julia NATSBridge Module]
JS_Module[JavaScript NATSBridge Module] JS_Module[JavaScript NATSBridge Module]
Python_Module[Python NATSBridge Module] Python_Module[Python NATSBridge Module]
Dart_Module[Dart NATSBridge Module]
MicroPython_Module[MicroPython NATSBridge Module] MicroPython_Module[MicroPython NATSBridge Module]
end end
@@ -80,6 +106,7 @@ flowchart TD
Julia_Module --> NATS_Client Julia_Module --> NATS_Client
JS_Module --> NATS_Client JS_Module --> NATS_Client
Python_Module --> NATS_Client Python_Module --> NATS_Client
Dart_Module --> NATS_Client
MicroPython_Module --> NATS_Client MicroPython_Module --> NATS_Client
NATS_Client --> NATS_Broker NATS_Client --> NATS_Broker
@@ -87,6 +114,7 @@ flowchart TD
Julia_Module --> File_Client Julia_Module --> File_Client
JS_Module --> File_Client JS_Module --> File_Client
Python_Module --> File_Client Python_Module --> File_Client
Dart_Module --> File_Client
MicroPython_Module --> File_Client MicroPython_Module --> File_Client
File_Client --> File_Server File_Client --> File_Server
@@ -94,6 +122,7 @@ flowchart TD
style Julia_Module fill:#e8f5e9,stroke:#4caf50 style Julia_Module fill:#e8f5e9,stroke:#4caf50
style JS_Module fill:#e3f2fd,stroke:#2196f3 style JS_Module fill:#e3f2fd,stroke:#2196f3
style Python_Module fill:#e3f2fd,stroke:#2196f3 style Python_Module fill:#e3f2fd,stroke:#2196f3
style Dart_Module fill:#fff0f6,stroke:#e91e63
style MicroPython_Module fill:#fce4ec,stroke:#e91e63 style MicroPython_Module fill:#fce4ec,stroke:#e91e63
style NATS_Broker fill:#fff3e0,stroke:#f57c00 style NATS_Broker fill:#fff3e0,stroke:#f57c00
style File_Server fill:#f3e5f5,stroke:#9c27b4 style File_Server fill:#f3e5f5,stroke:#9c27b4
@@ -159,8 +188,8 @@ flowchart TD
| **_build_envelope** | Build message envelope from payloads | All | | **_build_envelope** | Build message envelope from payloads | All |
| **_build_payload** | Build payload object from serialized data | All | | **_build_payload** | Build payload object from serialized data | All |
| **publish_message** | Publish message to NATS subject | All | | **publish_message** | Publish message to NATS subject | All |
| **fileserver_upload_handler** | Upload large payloads to HTTP server | Desktop | | **fileserver_upload_handler** | Upload large payloads to HTTP server | Desktop (Julia/JS/Python/Dart) |
| **fileserver_download_handler** | Download payloads from HTTP server | Desktop | | **fileserver_download_handler** | Download payloads from HTTP server | Desktop (Julia/JS/Python/Dart) |
### Data Flow ### Data Flow
@@ -275,8 +304,8 @@ end
|------|-------------|---------------|----------|-----------| |------|-------------|---------------|----------|-----------|
| `text` | Plain text string | UTF-8 bytes | Base64 | All | | `text` | Plain text string | UTF-8 bytes | Base64 | All |
| `dictionary` | JSON object | JSON string | Base64/JSON | All | | `dictionary` | JSON object | JSON string | Base64/JSON | All |
| `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop | | `arrowtable` | Apache Arrow IPC | Arrow IPC stream | Base64/arrow-ipc | Desktop (Julia/Python/Node.js/Dart) |
| `jsontable` | JSON array of objects | JSON string | Base64/json | All | | `jsontable` | JSON array of objects | JSON string | Base64/json | All (including Browser/Dart Web) |
| `image` | Binary image data | Raw bytes | Base64 | All | | `image` | Binary image data | Raw bytes | Base64 | All |
| `audio` | Binary audio data | Raw bytes | Base64 | All | | `audio` | Binary audio data | Raw bytes | Base64 | All |
| `video` | Binary video data | Raw bytes | Base64 | All | | `video` | Binary video data | Raw bytes | Base64 | All |
@@ -318,7 +347,10 @@ flowchart TD
| Platform | Size Threshold | Notes | | Platform | Size Threshold | Notes |
|----------|----------------|-------| |----------|----------------|-------|
| Desktop (Julia/JS/Python) | 500,000 bytes (0.5MB) | Default threshold | | Desktop (Julia/JS/Python/Dart) | 500,000 bytes (0.5MB) | Default threshold |
| Dart Desktop | 500,000 bytes (0.5MB) | Default threshold |
| Dart Flutter | 500,000 bytes (0.5MB) | Default threshold |
| Dart Web | 500,000 bytes (0.5MB) | Default threshold |
| MicroPython | 100,000 bytes (100KB) | Lower threshold for memory constraints | | MicroPython | 100,000 bytes (100KB) | Lower threshold for memory constraints |
### Transport Selection Flow ### Transport Selection Flow
@@ -405,21 +437,68 @@ end
JavaScript uses async/await for non-blocking I/O: JavaScript uses async/await for non-blocking I/O:
- **Class-based NATS Client**: Connection management - **Class-based NATS Client**: Connection management with `keepAlive` support
- **Module-level Utilities**: Serialization functions - **Module-level Utilities**: Serialization functions
- **Native ArrayBuffer**: Binary data handling - **Native ArrayBuffer**: Binary data handling (Browser) / Buffer (Node.js)
- **Fetch API**: HTTP file server communication - **Fetch API**: HTTP file server communication
- **Connection Pooling**: `NATSConnectionPool` for high-throughput scenarios
#### Node.js Implementation (natsbridge_ssr.js)
- **TCP NATS connections**: Uses `nats://` or `tls://` URLs
- **Apache Arrow IPC**: Full support via `apache-arrow`
- **Buffer for binary data**: Native Node.js Buffer handling
```javascript ```javascript
// Class-based NATS client // Class-based NATS client with keepAlive support
class NATSClient { class NATSClient {
constructor(url) { constructor(url, keepAlive = false) {
this.url = url; this.url = url;
this.connection = null; this.connection = null;
this.keepAlive = keepAlive;
} }
async connect() { async connect() {
if (this.connection) return this.connection;
this.connection = await nats.connect({ servers: this.url }); this.connection = await nats.connect({ servers: this.url });
return this.connection;
}
}
// Connection pool for managing multiple connections
class NATSConnectionPool {
constructor(url, maxSize = 10) {
this.url = url;
this.maxSize = maxSize;
this.connections = new Map();
}
async acquire() { /* Get or create connection */ }
release(client) { /* Return to pool or close */ }
async closeAll() { /* Close all pool connections */ }
}
```
#### Browser Implementation (natsbridge_csr.js)
- **WebSocket NATS connections**: Uses `ws://` or `wss://` URLs via `nats.ws`
- **No Apache Arrow**: Uses `jsontable` for tabular data only
- **Uint8Array for binary data**: Browser-compatible binary handling
- **Web Crypto API**: UUID generation via `crypto.getRandomValues()`
```javascript
// Class-based NATS client with keepAlive support
class NATSClient {
constructor(url, keepAlive = false) {
this.url = url; // ws:// or wss://
this.connection = null;
this.keepAlive = keepAlive;
}
async connect() {
if (this.connection) return this.connection;
this.connection = await nats.connect({ servers: this.url });
return this.connection;
} }
} }
``` ```
@@ -442,6 +521,60 @@ class NATSBridge:
self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL self.fileserver_url = fileserver_url or self.DEFAULT_FILESERVER_URL
``` ```
### Dart Architecture
Dart uses classes for stateful operations with async/await:
- **Class-based NATSBridge**: Encapsulated API
- **Data classes**: Structured data (MsgPayloadV1, MsgEnvelopeV1)
- **Async/await**: I/O operations
- **dart-arrow**: Arrow IPC support (Desktop/Flutter only)
- **HTTP package**: HTTP file server communication
- **nats package**: NATS client with WebSocket support (Dart Web)
```dart
class NATSBridge {
static const DEFAULT_SIZE_THRESHOLD = 500000;
final String brokerUrl;
final String fileserverUrl;
NATSBridge({
this.brokerUrl = 'nats://localhost:4222',
this.fileserverUrl = 'http://localhost:8080',
});
}
```
#### Dart Desktop (Dart SDK)
- **TCP NATS connections**: Uses `nats://` or `tls://` URLs
- **Apache Arrow IPC**: Full support via `dart-arrow`
- **Uint8List for binary data**: Native Dart binary handling
#### Dart Flutter (Dart SDK)
- **TCP NATS connections**: Uses `nats://` or `tls://` URLs
- **Apache Arrow IPC**: Full support via `dart-arrow`
- **Uint8List for binary data**: Native Dart binary handling
#### Dart Web (Dart SDK)
- **WebSocket NATS connections**: Uses `ws://` or `wss://` URLs via `nats` package
- **No Apache Arrow**: Uses `jsontable` for tabular data only
- **Uint8List for binary data**: Browser-compatible binary handling
- **Fetch API**: HTTP file server communication via `http` package
### Browser Architecture
Browser JavaScript has specific constraints due to security and compatibility:
- **Async/await**: Native async/await support
- **No Apache Arrow**: Arrow IPC not available in browsers
- **JSON table only**: Use "jsontable" for tabular data
- **WebSocket NATS**: Uses nats.ws for browser-compatible NATS connections
- **Fetch API**: HTTP file server communication via fetch
### MicroPython Architecture ### MicroPython Architecture
MicroPython has significant constraints: MicroPython has significant constraints:
@@ -598,7 +731,7 @@ MAX_PAYLOAD_SIZE = 50_000 # 50KB hard limit
|-----------|---------|-------| |-----------|---------|-------|
| NATS Server | 1 instance | Single node for development | | NATS Server | 1 instance | Single node for development |
| File Server | 1 instance | HTTP server for large payloads | | File Server | 1 instance | HTTP server for large payloads |
| Client Memory | 50MB | Desktop platforms | | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) |
| Client Memory | 256KB | MicroPython devices | | Client Memory | 256KB | MicroPython devices |
### Environment Variables ### Environment Variables
@@ -693,7 +826,7 @@ flowchart TD
| Version | Supported Platforms | | Version | Supported Platforms |
|---------|---------------------| |---------|---------------------|
| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, MicroPython 1.19+ | | v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, Dart 2.17+, MicroPython 1.19+ |
--- ---
@@ -701,16 +834,85 @@ flowchart TD
| Date | Version | Changes | | Date | Version | Changes |
|------|---------|---------| |------|---------|---------|
| 2026-03-15 | 1.1.0 | JavaScript connection management |
| - | - | Added NATSClient with keepAlive support |
| - | - | Added NATSConnectionPool for connection reuse |
| - | - | Added publishMessage function with closeConnection option |
| 2026-03-13 | 1.0.0 | Initial architecture documentation | | 2026-03-13 | 1.0.0 | Initial architecture documentation |
--- ---
## References ## 16. References
- [`docs/requirements.md`](./requirements.md) - Business requirements and user stories ### 16.1 Documentation Artifacts
- [`docs/spec.md`](./spec.md) - Technical specification and contracts
- [`src/NATSBridge.jl`](../src/NATSBridge.jl) - Ground truth implementation | Document | Purpose | Specification Traceability | UI Specification Traceability | Requirement ID(s) |
- [`README.md`](../README.md) - Project overview |----------|---------|---------------------------|------------------------------|-------------------|
| [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 | - | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/specification.md`](./specification.md) | Technical contract for NATSBridge | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | - | All UI components and interactions | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/validation.md`](./validation.md) | CI/CD validation rules | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/runbook.md`](./runbook.md) | Operational runbook | specification.md:2-19 (all sections) | - | FR-001 through FR-014, NFR-101 through NFR-405 |
### 16.2 Implementation Files
| File | Platform | Features | Specification Traceability | Requirement ID(s) |
|------|----------|----------|---------------------------|-------------------|
| [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge_ssr.js`](../src/natsbridge_ssr.js) | Node.js | Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge_csr.js`](../src/natsbridge_csr.js) | Browser | JSON table only, WebSocket NATS | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge.py`](../src/natsbridge.py) | Python | Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge.dart`](../src/natsbridge.dart) | Dart | Full feature set, Arrow IPC, async/await | specification.md:2-19 (all sections) | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge_mpy.py`](../src/natsbridge_mpy.py) | MicroPython | Limited to direct transport | specification.md:2-19 (all sections) | FR-005, FR-006, FR-012 |
### 16.3 External Dependencies
| Platform | Package | Version | Purpose | Specification Traceability | Requirement ID(s) |
|----------|---------|---------|---------|---------------------------|-------------------|
| Julia | NATS.jl | Latest | NATS client | specification.md:11 | FR-013, FR-014, NFR-201 |
| Julia | JSON.jl | Latest | JSON serialization | specification.md:11 | FR-012, NFR-101, NFR-102 |
| Julia | Arrow.jl | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 |
| Julia | HTTP.jl | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 |
| Julia | UUIDs.jl | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 |
| Node.js | nats | Latest | NATS client (TCP) | specification.md:11 | FR-013, FR-014 |
| Node.js | node-fetch | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 |
| Browser | nats.ws | Latest | NATS client (WebSocket) | specification.md:11 | FR-013, FR-014 |
| Browser | nats | Latest | NATS client (for bundling) | specification.md:11 | FR-013, FR-014 |
| Python | nats-py | Latest | NATS client | specification.md:11 | FR-013, FR-014 |
| Python | aiohttp | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 |
| Python | pyarrow | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 |
| Dart | nats | Latest | NATS client | specification.md:11 | FR-013, FR-014 |
| Dart | http | Latest | HTTP file server | specification.md:11 | FR-008, FR-009 |
| Dart | uuid | Latest | UUID generation | specification.md:11 | FR-011, NFR-401 |
| Dart | dart-arrow | Latest | Arrow IPC support | specification.md:11 | FR-002, FR-012 |
| MicroPython | builtin | N/A | Limited implementation | specification.md:11 | FR-005, FR-006, FR-012 |
---
## 17. Change Log
| Date | Version | Changes | Specification Reference |
|------|---------|---------|------------------------|
| 2026-03-23 | 1.1.0 | Updated to ASG Framework architecture guidelines | specification.md:2-19 (all sections) |
| 2026-03-15 | 1.1.0 | JavaScript connection management | specification.md:2-19 (all sections) |
| 2026-03-13 | 1.0.0 | Initial architecture documentation | specification.md:2-19 (all sections) |
---
## 18. Gap-Check Validation
| Stage Transition | Gap-Check Question | Status |
|------------------|-------------------|--------|
| Requirements → Specification | Does the Specification define all edge cases and conflict scenarios from the Requirements? | ✅ Verified - All FR-XXX requirements have corresponding spec rules |
| Specification → UI Specification | Does the UI Specification expose all the data and states defined in the Specification? | ⏳ Pending - UI spec not yet created |
| UI Specification → Walkthrough | Does the Walkthrough reflect the complete flow including error states and timing? | ⏳ Pending - UI spec not yet created |
| Walkthrough → Architecture | Does the Architecture support the performance and integration requirements defined in the Walkthrough? | ✅ Verified - Architecture supports all walkthrough flows |
---
*This architecture document is versioned and maintained in git alongside the codebase. All implementations must adhere to this architecture.*
--- ---

View File

@@ -1,33 +1,38 @@
# Requirements Document: NATSBridge # Requirements Document: NATSBridge
**Version**: 1.0.0 **Version**: 1.0.0
**Date**: 2026-03-13 **Date**: 2026-03-23
**Status**: Active **Status**: Active
**Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl) **Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl)
--- ---
## Executive Summary ## 1. Business Context & Success Metrics
NATSBridge is a cross-platform, bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, and **MicroPython** applications using NATS as the message bus. The system implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB) by uploading them to an HTTP file server instead of sending raw binary data over NATS. ### 1.1 Business Goal
--- NATSBridge is a cross-platform, bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, and **MicroPython** applications using NATS as the message bus. The system implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB) by uploading them to an HTTP file server instead of sending raw binary data over NATS.
## Business Goals ### 1.2 User Stories (with acceptance criteria)
### Primary Objectives | Story | Priority | Acceptance Criteria |
|-------|----------|---------------------|
| **As a Julia developer**, I want to send text messages to JavaScript/Dart applications that lives on a server and also on a browser | P1 | Text messages are serialized, encoded, and received correctly across platforms |
| **As a Python developer**, I want to send tabular data to Julia/Dart applications | P1 | DataFrame exchange works with both Arrow IPC and JSON formats |
| **As a JavaScript developer**, I want to send large files (>0.5MB) from JavaScript applications that lives on a server and also on a browser to other applications | P1 | Large files are automatically uploaded to file server and URLs are sent via NATS |
| **As a Dart developer**, I want to send text messages to other platforms | P1 | Text messages are serialized, encoded, and received correctly across platforms |
| **As a Dart developer**, I want to send dictionary data to other platforms | P1 | JSON-serializable data is exchanged correctly |
| **As a Dart developer**, I want to send tabular data (List<Map>) to other platforms | P1 | JSON table format exchange works with Arrow IPC on desktop |
| **As a Dart developer**, I want to send large files (>0.5MB) | P1 | Large files are automatically uploaded to file server and URLs are sent via NATS |
| **As a MicroPython developer**, I want to send sensor data with minimal memory usage | P1 | Direct transport works for payloads <100KB on memory-constrained devices |
| **As a developer**, I want to send mixed-content messages (text + image + file) | P1 | NATSBridge accepts list of (dataname, data, type) tuples and handles each payload appropriately |
| **As a developer**, I want to receive multi-payload messages | P1 | NATSBridge returns payloads as list of tuples with correct types preserved |
| **As a developer**, I want to use Plik as the file server | P2 | Plik one-shot upload mode is supported with upload ID and token handling |
| **As a developer**, I want to use custom HTTP file servers | P2 | Handler function abstraction allows plugging in AWS S3 or custom implementations |
| **As a developer**, I want automatic retry on file server download failures | P1 | Exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) |
| **As a developer**, I want message tracing across distributed systems | P1 | Correlation ID is propagated through all message processing steps |
1. **Cross-Platform Interoperability**: Enable seamless data exchange between Julia, JavaScript (for both Server-Side rendering and Client-Side rendering webapp), Python, and MicroPython applications without platform-specific barriers. ### 1.3 KPIs & Targets
2. **Efficient Large Payload Handling**: Implement intelligent transport selection based on payload size:
- **Direct Transport**: Small payloads (<0.5MB) sent directly via NATS
- **Link Transport**: Large payloads (≥0.5MB) uploaded to HTTP file server, URL sent via NATS
3. **Unified API Across Platforms**: Provide consistent `smartsend()` and `smartreceive()` functions across all supported platforms while maintaining idiomatic implementations.
4. **Developer Productivity**: Reduce onboarding time and simplify integration through comprehensive documentation and test examples.
### Success Metrics
| Metric | Target | Measurement Method | | Metric | Target | Measurement Method |
|--------|--------|-------------------| |--------|--------|-------------------|
@@ -40,90 +45,22 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
--- ---
## User Stories ## 2. Technical Boundaries
### Core Functionality ### 2.1 In Scope
| Story | Priority | Acceptance Criteria | | Feature | Description |
|-------|----------|---------------------| |---------|-------------|
| **As a Julia developer**, I want to send text messages to JavaScript applications that lives on a server and also on a browser | P1 | Text messages are serialized, encoded, and received correctly across platforms | | Cross-platform interoperability | Seamless data exchange between Julia, JavaScript, Python, Dart, and MicroPython |
| **As a Python developer**, I want to send tabular data to Julia applications | P1 | DataFrame exchange works with both Arrow IPC and JSON formats | | Intelligent transport selection | Direct transport (<0.5MB) vs Link transport (≥0.5MB) based on payload size |
| **As a JavaScript developer**, I want to send large files (>0.5MB) from JavaScript applications that lives on a server and also on a browser to other applications | P1 | Large files are automatically uploaded to file server and URLs are sent via NATS | | Unified API | Consistent `smartsend()` and `smartreceive()` functions across all platforms |
| **As a MicroPython developer**, I want to send sensor data with minimal memory usage | P1 | Direct transport works for payloads <100KB on memory-constrained devices | | Multi-payload support | List of (dataname, data, type) tuples with appropriate handling |
| File server integration | Plik one-shot upload and custom HTTP server support |
| Reliability features | Exponential backoff retry and correlation ID propagation |
| Message serialization | Converts data types to binary format (Base64, JSON, Arrow IPC) |
| NATS communication | Publishing and subscription via NATS subjects |
### Multi-Payload Support ### 2.2 Out of Scope
| Story | Priority | Acceptance Criteria |
|-------|----------|---------------------|
| **As a developer**, I want to send mixed-content messages (text + image + file) | P1 | NATSBridge accepts list of (dataname, data, type) tuples and handles each payload appropriately |
| **As a developer**, I want to receive multi-payload messages | P1 | NATSBridge returns payloads as list of tuples with correct types preserved |
### File Server Integration
| Story | Priority | Acceptance Criteria |
|-------|----------|---------------------|
| **As a developer**, I want to use Plik as the file server | P2 | Plik one-shot upload mode is supported with upload ID and token handling |
| **As a developer**, I want to use custom HTTP file servers | P2 | Handler function abstraction allows plugging in AWS S3 or custom implementations |
### Reliability Features
| Story | Priority | Acceptance Criteria |
|-------|----------|---------------------|
| **As a developer**, I want automatic retry on file server download failures | P1 | Exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) |
| **As a developer**, I want message tracing across distributed systems | P1 | Correlation ID is propagated through all message processing steps |
---
## Non-Functional Requirements
### Performance Requirements
| Requirement | Specification | Test Method |
|-------------|---------------|-------------|
| Message serialization overhead | <50ms for 10KB payload | Benchmark tests |
| Message deserialization overhead | <50ms for 10KB payload | Benchmark tests |
| NATS connection establishment | <100ms | Connection pool benchmarks |
| File upload latency | <1s for 0.5MB file | Integration tests |
| File download latency | <1s for 0.5MB file | Integration tests |
### Scalability Requirements
| Requirement | Specification |
|-------------|---------------|
| Concurrent connections | Support 100+ simultaneous NATS connections |
| Message throughput | Handle 1000+ messages/second per instance |
| File server scalability | Support horizontal scaling of file server backend |
### Reliability Requirements
| Requirement | Specification |
|-------------|---------------|
| Message delivery | At-least-once delivery semantics via NATS |
| File server availability | Graceful degradation when file server is unavailable |
| Connection recovery | Auto-reconnect on NATS connection failure |
### Security Requirements
| Requirement | Specification |
|-------------|---------------|
| Payload integrity | SHA-256 checksum support via metadata |
| Transport security | TLS support for NATS connections |
| File server security | Authentication token for file uploads |
### Compatibility Requirements
| Platform | Minimum Version | Notes |
|----------|-----------------|-------|
| Julia | 1.7+ | Arrow.jl required for arrowtable support |
| Node.js | 16+ | nats.js required |
| Python | 3.8+ | pyarrow required for arrowtable support |
| MicroPython | 1.19+ | Limited to direct transport |
---
## Out of Scope
### Phase 1 (Current Implementation)
| Feature | Reason | | Feature | Reason |
|---------|--------| |---------|--------|
@@ -133,94 +70,173 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
| Persistent message queues | NATS request-reply pattern sufficient | | Persistent message queues | NATS request-reply pattern sufficient |
| Advanced routing rules | Simple NATS subject matching sufficient | | Advanced routing rules | Simple NATS subject matching sufficient |
### Future Considerations ### 2.3 Dependencies
| Feature | Future Phase | | Platform | Package | Version |
|---------|--------------| |----------|---------|---------|
| JetStream streams and consumers | Phase 2 | | Julia | NATS.jl | Latest stable |
| Message TTL and dead-letter queues | Phase 3 | | Julia | JSON.jl | Latest stable |
| Message tracing with OpenTelemetry | Phase 3 | | Julia | Arrow.jl | Latest stable |
| Rate limiting and quota management | Phase 4 | | Julia | HTTP.jl | Latest stable |
| Julia | UUIDs.jl | Latest stable |
| Node.js | nats | Latest stable |
| Node.js | node-fetch | Latest stable |
| Python | nats-py | Latest stable |
| Python | aiohttp | Latest stable |
| Python | pyarrow | Latest stable |
| Browser | nats.ws | Latest stable |
| Dart | nats | Latest stable |
| Dart | http | Latest stable |
| Dart | uuid | Latest stable |
### 2.4 Platform Compatibility
| Platform | Minimum Version | Notes |
|----------|-----------------|-------|
| Julia | 1.7+ | Arrow.jl required for arrowtable support |
| Node.js | 16+ | nats.js required, Arrow IPC supported |
| Python | 3.8+ | pyarrow required for arrowtable support |
| Browser | Latest | No Arrow IPC (uses jsontable only) |
| Dart | 2.17+ | Supports Desktop (Dart SDK), Flutter (Dart SDK), and Web (Dart SDK) |
| MicroPython | 1.19+ | Limited to direct transport |
--- ---
## Boundary Definitions ## 3. Functional Requirements (FR)
### What NATSBridge Handles | ID | Requirement | Description |
|----|-------------|-------------|
| Function | Description | | **FR-001** | Cross-platform text messaging | System shall allow users to send text messages between Julia, JavaScript, Python, and MicroPython applications |
|----------|-------------| | **FR-002** | Cross-platform tabular data | System shall support DataFrame exchange between Julia and Python applications using Arrow IPC format |
| Message serialization | Converts data types to binary format | | **FR-003** | Large file handling | System shall automatically detect payloads ≥0.5MB and upload them to HTTP file server instead of sending via NATS |
| Message encoding | Base64, JSON, Arrow IPC encoding | | **FR-004** | Direct transport for small payloads | System shall send payloads <0.5MB directly via NATS without file server upload |
| Transport selection | Direct vs link based on size threshold | | **FR-005** | MicroPython support | System shall support payloads <100KB on MicroPython devices using direct transport |
| NATS publishing | Publishes messages to NATS subjects | | **FR-006** | Multi-payload messages | System shall accept and process lists of (dataname, data, type) tuples |
| NATS subscription | Receives and processes NATS messages | | **FR-007** | Payload type preservation | System shall preserve payload types when returning multi-payload messages |
| File server upload | Uploads large payloads to HTTP server | | **FR-008** | Plik file server integration | System shall support Plik one-shot upload mode with upload ID and token handling |
| File server download | Downloads payloads from HTTP server with retry | | **FR-009** | Custom file server support | System shall provide handler function abstraction for custom HTTP file server implementations |
| Correlation ID generation | Creates and propagates UUIDs | | **FR-010** | Exponential backoff retry | System shall implement exponential backoff with configurable retries (default: 5, base_delay: 100ms, max_delay: 5000ms) for file server download failures |
| Data deserialization | Converts binary format back to native types | | **FR-011** | Correlation ID propagation | System shall propagate correlation IDs through all message processing steps |
| **FR-012** | Message serialization | System shall serialize data types using Base64, JSON, or Arrow IPC encoding |
### What NATSBridge Does NOT Handle | **FR-013** | NATS publishing | System shall publish messages to NATS subjects |
| **FR-014** | NATS subscription | System shall receive and process NATS messages |
| Function | Handled By |
|----------|------------|
| NATS server management | External NATS deployment |
| File server management | External HTTP server deployment |
| Application business logic | Application code using NATSBridge |
| Message encryption | Application layer |
| Message compression | Application layer |
| Authentication/Authorization | NATS server configuration |
--- ---
## Payload Type Requirements ## 4. Non-Functional Requirements (NFRs)
### Supported Payload Types ### 4.1 Performance & Scalability
| Type | Julia | JavaScript | Python | MicroPython | Description | | ID | Requirement | Specification | Test Method |
|------|-------|------------|--------|-------------|-------------| |----|-------------|---------------|-------------|
| `text` | `String` | `string` | `str` | `str` | Plain text strings | | **NFR-101** | Message serialization overhead | <50ms for 10KB payload | Benchmark tests |
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `dict` | JSON-serializable data | | **NFR-102** | Message deserialization overhead | <50ms for 10KB payload | Benchmark tests |
| `arrowtable` | `DataFrame`, `Arrow.Table` | `Array<Object>` | `pandas.DataFrame` | ❌ | Tabular data (Arrow IPC) | | **NFR-103** | NATS connection establishment | <100ms | Connection pool benchmarks |
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ⚠️ | Tabular data (JSON) | | **NFR-104** | File upload latency | <1s for 0.5MB file | Integration tests |
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Image binary data | | **NFR-105** | File download latency | <1s for 0.5MB file | Integration tests |
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Audio binary data | | **NFR-106** | Concurrent connections | Support 100+ simultaneous NATS connections | Scale testing |
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `bytearray` | Video binary data | | **NFR-107** | Message throughput | Handle 1000+ messages/second per instance | Load testing |
| `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | `bytearray` | Generic binary data | | **NFR-108** | File server scalability | Support horizontal scaling of file server backend | Architecture review |
### Encoding Requirements ### 4.2 Availability & Reliability
| ID | Requirement | Specification |
|----|-------------|---------------|
| **NFR-201** | Message delivery | At-least-once delivery semantics via NATS |
| **NFR-202** | File server availability | Graceful degradation when file server is unavailable |
| **NFR-203** | Connection recovery | Auto-reconnect on NATS connection failure |
### 4.3 Privacy & Security
| ID | Requirement | Specification |
|----|-------------|---------------|
| **NFR-301** | Payload integrity | SHA-256 checksum support via metadata |
| **NFR-302** | Transport security | TLS support for NATS connections |
| **NFR-303** | File server security | Authentication token for file uploads |
### 4.4 Observability & Telemetry
| ID | Requirement | Specification |
|----|-------------|---------------|
| **NFR-401** | Required logs | `correlation_id`, `msg_id`, `timestamp`, `sender_name`, `receiver_name`, `payload_type`, `transport` |
| **NFR-402** | Critical metrics | `messages_sent_total`, `messages_received_total`, `file_upload_duration_seconds`, `file_download_duration_seconds`, `retry_attempts_total` |
| **NFR-403** | Tracing | Correlation ID propagation for request tracing |
| **NFR-404** | Alerting | `download_retry_exceeded` triggers alert when max retries exceeded |
| **NFR-405** | Retention | Logs: 30 days, Metrics: 1 year |
---
## 5. Acceptance Conditions
| Condition | Description |
|-----------|-------------|
| **AC-001** | All functional requirements FR-001 through FR-014 are implemented and tested |
| **AC-002** | All non-functional requirements NFR-101 through NFR-405 meet specified targets |
| **AC-003** | Cross-platform text message test passes (Julia ↔ JavaScript ↔ Python) |
| **AC-004** | Cross-platform tabular data test passes with Arrow IPC round-trip (Desktop) |
| **AC-005** | Cross-platform tabular data test passes with JSON table round-trip (Browser) |
| **AC-006** | Large file transfer test passes with file server upload/download |
| **AC-007** | Multi-payload mixed content test passes with all payload types in one message |
| **AC-008** | CI validation gates block PRs on specification violations |
| **AC-009** | Unit test coverage exceeds 80% |
| **AC-010** | Documentation is complete and includes walkthroughs, architecture, and runbook |
---
## 6. Payload Type Requirements
### 6.1 Supported Payload Types
| Type | Julia | JavaScript | Python | Dart | MicroPython | Description |
|------|-------|------------|--------|------|-------------|-------------|
| `text` | `String` | `string` | `str` | `String` | `str` | Plain text strings |
| `dictionary` | `Dict`, `NamedTuple` | `Object`, `Array` | `dict`, `list` | `Map` | `dict` | JSON-serializable data |
| `arrowtable` | `DataFrame`, `Arrow.Table` | ❌ (Browser), ✅ (Node.js) | `pandas.DataFrame` | `List<Map>` (Desktop), `List<dynamic>` (Flutter) | ❌ | Tabular data (Arrow IPC) |
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | `List<Map>` | ⚠️ | Tabular data (JSON) - **Only table type in Browser** |
| `image` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `Uint8List` | `bytearray` | Image binary data |
| `audio` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `Uint8List` | `bytearray` | Audio binary data |
| `video` | `Vector{UInt8}` | `Uint8Array`, `Buffer` | `bytes` | `Uint8List` | `bytearray` | Video binary data |
| `binary` | `Vector{UInt8}`, `IOBuffer` | `Uint8Array`, `Buffer` | `bytes`, `bytearray` | `Uint8List` | `bytearray` | Generic binary data |
### 6.2 Encoding Requirements
| Payload Type | Encoding Method | Notes | | Payload Type | Encoding Method | Notes |
|--------------|-----------------|-------| |--------------|-----------------|-------|
| `text` | UTF-8 → Base64 | Text must be String type | | `text` | UTF-8 → Base64 | Text must be String type |
| `dictionary` | JSON → Base64 | JSON.jl for Julia | | `dictionary` | JSON → Base64 | JSON.jl for Julia |
| `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow | | `arrowtable` | Arrow IPC → Base64 | Requires Arrow.jl/pyarrow (Desktop only) |
| `jsontable` | JSON → Base64 | Human-readable format | | `jsontable` | JSON → Base64 | Human-readable format - **Browser uses this only** |
| `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved | | `image`/`audio`/`video`/`binary` | Direct → Base64 | Binary data preserved |
--- ---
## Size Threshold Requirements ## 7. Size Threshold Requirements
### Direct Transport Threshold ### 7.1 Direct Transport Threshold
| Platform | Threshold | Notes | | Platform | Threshold | Notes |
|----------|-----------|-------| |----------|-----------|-------|
| Desktop (Julia/JS/Python) | 0.5MB | Default size threshold | | Desktop (Julia/JS/Python/Dart) | 0.5MB | Default size threshold |
| Dart Desktop | 0.5MB | Default size threshold |
| Dart Flutter | 0.5MB | Default size threshold |
| Dart Web | 0.5MB | Default size threshold |
| MicroPython | 100KB | Lower threshold for memory constraints | | MicroPython | 100KB | Lower threshold for memory constraints |
### Maximum Payload Size ### 7.2 Maximum Payload Size
| Platform | Maximum | Notes | | Platform | Maximum | Notes |
|----------|---------|-------| |----------|---------|-------|
| Desktop | Unlimited | Limited by NATS server configuration | | Desktop | Unlimited | Limited by NATS server configuration |
| Dart Desktop | Unlimited | Limited by NATS server configuration |
| Dart Flutter | Unlimited | Limited by NATS server configuration |
| Dart Web | Unlimited | Limited by NATS server configuration |
| MicroPython | 50KB | Hard limit due to 256KB-1MB memory | | MicroPython | 50KB | Hard limit due to 256KB-1MB memory |
--- ---
## Message Envelope Requirements ## 8. Message Envelope Requirements
### Required Fields ### 8.1 Required Fields
| Field | Type | Purpose | | Field | Type | Purpose |
|-------|------|---------| |-------|------|---------|
@@ -239,7 +255,7 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
| `metadata` | Dict | Message-level metadata | | `metadata` | Dict | Message-level metadata |
| `payloads` | Array | List of payload objects | | `payloads` | Array | List of payload objects |
### Payload Fields ### 8.2 Payload Fields
| Field | Type | Purpose | | Field | Type | Purpose |
|-------|------|---------| |-------|------|---------|
@@ -254,9 +270,9 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
--- ---
## Error Handling Requirements ## 9. Error Handling Requirements
### Error Codes ### 9.1 Error Codes
| Error | Condition | Response | | Error | Condition | Response |
|-------|-----------|----------| |-------|-----------|----------|
@@ -266,7 +282,7 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
| `Unknown transport` | Invalid transport type | Throw error | | `Unknown transport` | Invalid transport type | Throw error |
| `NATS connection failed` | NATS unavailable | Throw error | | `NATS connection failed` | NATS unavailable | Throw error |
### Exception Handling ### 9.2 Exception Handling
| Scenario | Handler | | Scenario | Handler |
|----------|---------| |----------|---------|
@@ -277,9 +293,9 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
--- ---
## Testing Requirements ## 10. Testing Requirements
### Unit Tests ### 10.1 Unit Tests
| Test Category | Coverage | Files | | Test Category | Coverage | Files |
|---------------|----------|-------| |---------------|----------|-------|
@@ -289,20 +305,21 @@ NATSBridge is a cross-platform, bi-directional data bridge that enables seamless
| File server upload | Plik integration | Platform-specific | | File server upload | Plik integration | Platform-specific |
| File server download | Exponential backoff | Platform-specific | | File server download | Exponential backoff | Platform-specific |
### Integration Tests ### 10.2 Integration Tests
| Test Scenario | Success Criteria | | Test Scenario | Success Criteria |
|-------------|-----------------| |-------------|-----------------|
| Cross-platform text message | Julia ↔ JavaScript ↔ Python | | Cross-platform text message | Julia ↔ JavaScript ↔ Python |
| Cross-platform tabular data | Arrow IPC round-trip | | Cross-platform tabular data (Desktop) | Arrow IPC round-trip |
| Cross-platform tabular data (Browser) | JSON table round-trip |
| Large file transfer | File server upload/download | | Large file transfer | File server upload/download |
| Multi-payload mixed content | All payload types in one message | | Multi-payload mixed content | All payload types in one message |
--- ---
## API Contract ## 11. API Contract
### smartsend Signature ### 11.1 smartsend Signature
```julia ```julia
function smartsend( function smartsend(
@@ -326,7 +343,7 @@ function smartsend(
)::Tuple{msg_envelope_v1, String} )::Tuple{msg_envelope_v1, String}
``` ```
### smartreceive Signature ### 11.2 smartreceive Signature
```julia ```julia
function smartreceive( function smartreceive(
@@ -340,44 +357,18 @@ function smartreceive(
--- ---
## Dependencies ## 12. Deployment Requirements
### Required Dependencies ### 12.1 Minimum Infrastructure
| Platform | Package | Version |
|----------|---------|---------|
| Julia | NATS.jl | Latest stable |
| Julia | JSON.jl | Latest stable |
| Julia | Arrow.jl | Latest stable |
| Julia | HTTP.jl | Latest stable |
| Julia | UUIDs.jl | Latest stable |
| Node.js | nats | Latest stable |
| Node.js | node-fetch | Latest stable |
| Python | nats-py | Latest stable |
| Python | aiohttp | Latest stable |
| Python | pyarrow | Latest stable |
### Optional Dependencies
| Platform | Package | Use Case |
|----------|---------|----------|
| Julia | DataFrames.jl | DataFrame support for arrowtable |
| Python | pandas | DataFrame support for arrowtable |
---
## Deployment Requirements
### Minimum Infrastructure
| Component | Minimum | Notes | | Component | Minimum | Notes |
|-----------|---------|-------| |-----------|---------|-------|
| NATS Server | 1 instance | Single node for development | | NATS Server | 1 instance | Single node for development |
| File Server | 1 instance | HTTP server for large payloads | | File Server | 1 instance | HTTP server for large payloads |
| Client Memory | 50MB | Desktop platforms | | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) |
| Client Memory | 256KB | MicroPython devices | | Client Memory | 256KB | MicroPython devices |
### Environment Variables ### 12.2 Environment Variables
| Variable | Default | Description | | Variable | Default | Description |
|----------|---------|-------------| |----------|---------|-------------|
@@ -387,34 +378,42 @@ function smartreceive(
--- ---
## Versioning ## 13. Versioning
### Current Version ### 13.1 Current Version
- **Major**: 1 (Breaking changes require major version bump) - **Major**: 1 (Breaking changes require major version bump)
- **Minor**: 0 (Feature additions) - **Minor**: 0 (Feature additions)
- **Patch**: 0 (Bug fixes) - **Patch**: 0 (Bug fixes)
### Version Compatibility ### 13.2 Version Compatibility
| Version | Supported Platforms | | Version | Supported Platforms |
|---------|---------------------| |---------|---------------------|
| v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, MicroPython 1.19+ | | v1.0.x | Julia 1.7+, Node.js 16+, Python 3.8+, Dart 2.17+, Browser (latest), MicroPython 1.19+ |
--- ---
## Change Log ## 14. Change Log
| Date | Version | Changes | | Date | Version | Changes |
|------|---------|---------| |------|---------|---------|
| 2026-03-13 | 1.0.0 | Initial requirements document | | 2026-03-23 | 1.0.0 | Updated to ASG Framework requirements structure |
--- ---
## References ## 15. References
- [`src/NATSBridge.jl`](../src/NATSBridge.jl) - Ground truth implementation - [`src/NATSBridge.jl`](../src/NATSBridge.jl) - Ground truth implementation (Julia)
- [`src/natsbridge_ssr.js`](../src/natsbridge_ssr.js) - Server-side JavaScript implementation
- [`src/natsbridge_csr.js`](../src/natsbridge_csr.js) - Client-side JavaScript implementation
- [`src/natsbridge.py`](../src/natsbridge.py) - Python implementation
- [`src/natsbridge.dart`](../src/natsbridge.dart) - Dart implementation
- [`src/natsbridge_mpy.py`](../src/natsbridge_mpy.py) - MicroPython implementation
- [`README.md`](../README.md) - Project overview - [`README.md`](../README.md) - Project overview
- [`docs/specification.md`](./specification.md) - Technical specification
- [`docs/ui-specification.md`](./ui-specification.md) - UI specification
- [`docs/walkthrough.md`](./walkthrough.md) - End-to-end walkthrough
- [`docs/architecture.md`](./architecture.md) - Architecture documentation - [`docs/architecture.md`](./architecture.md) - Architecture documentation
- [`docs/implementation.md`](./implementation.md) - Implementation details - [`docs/validation.md`](./validation.md) - Validation and CI/CD
- [`docs/walkthrough.md`](./walkthrough.md) - Usage examples - [`docs/runbook.md`](./runbook.md) - Operational runbook

View File

@@ -1,16 +1,16 @@
# Specification: NATSBridge # Specification: NATSBridge
**Version**: 1.0.0 **Version**: 1.1.0
**Date**: 2026-03-13 **Date**: 2026-03-23
**Status**: Active **Status**: Active
**Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl) **Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl)
**Specification Format**: JSON Schema + AsyncAPI **Specification Format**: JSON Schema + AsyncAPI
--- ---
## Executive Summary ## 1. Technical Contract Overview
This document defines the **technical contract** for NATSBridge - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, and **MicroPython** applications using NATS as the message bus. This document defines the **technical contract** for NATSBridge - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, and **MicroPython** applications using NATS as the message bus.
This specification serves as the single source of truth for: This specification serves as the single source of truth for:
- **Inputs**: What data structures are accepted by `smartsend()` - **Inputs**: What data structures are accepted by `smartsend()`
@@ -18,8 +18,33 @@ This specification serves as the single source of truth for:
- **Data Shapes**: Exact field names, types, and constraints - **Data Shapes**: Exact field names, types, and constraints
- **Error Codes**: Standardized error responses for failure scenarios - **Error Codes**: Standardized error responses for failure scenarios
### 1.1 Requirements Traceability
| Specification Section | Requirement ID(s) | Description |
|----------------------|-------------------|-------------|
| Section 2 (Message Envelope) | FR-012, FR-013, NFR-101, NFR-102 | Message envelope structure and validation |
| Section 3 (Payload Schema) | FR-001, FR-002, FR-003, FR-004, NFR-101, NFR-102 | Payload structure and field definitions |
| Section 4 (Payload Format) | FR-006, FR-007 | Tuple format for smartsend() |
| Section 5 (Enumerations) | FR-003, FR-004, FR-006, NFR-101 | Enumerations for transport and encoding |
| Section 6 (Transport Protocols) | FR-003, FR-004, NFR-104, NFR-105 | Direct and link transport protocols |
| Section 7 (Size Thresholds) | FR-004, FR-005, NFR-104, NFR-105 | Size thresholds for transport selection |
| Section 8 (NATS Subject Convention) | FR-013, FR-014 | NATS subject naming patterns |
| Section 9 (Error Handling) | FR-010, FR-011, NFR-201, NFR-202, NFR-203 | Error codes and exception handling |
| Section 10 (Serialization Rules) | FR-001, FR-002, FR-003, FR-012, NFR-101, NFR-102 | Serialization and encoding rules |
| Section 11 (API Contract) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Function signatures for all platforms |
| Section 12 (File Server Interface) | FR-008, FR-009, FR-010 | Upload and download handler contracts |
| Section 13 (Platform-Specific Constraints) | FR-005, FR-006, NFR-106, NFR-107 | Platform-specific feature support |
| Section 14 (Implementation Files) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007 | Implementation file mapping |
| Section 15 (Message Flow) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Mermaid diagrams for send/receive flows |
| Section 16 (Validation Rules) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Envelope and payload validation rules |
| Section 17 (Test Contracts) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Unit and integration test scenarios |
| Section 18 (Dependencies) | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 | Platform-specific dependencies |
| Section 19 (Change Log) | N/A | Version history and changes |
--- ---
## 2. Message Envelope Schema
## Specification Versioning ## Specification Versioning
| Component | Version | Notes | | Component | Version | Notes |
@@ -65,22 +90,22 @@ This specification serves as the single source of truth for:
### Field Definitions ### Field Definitions
| Field | Type | Required | Validation | Description | | Field | Type | Required | Validation | Description | Requirement ID |
|-------|------|----------|------------|-------------| |-------|------|----------|------------|-------------|----------------|
| `correlation_id` | `string` | Yes | UUID v4 format | Track message flow across distributed systems | | `correlation_id` | `string` | Yes | UUID v4 format | Track message flow across distributed systems | FR-011, NFR-401 |
| `msg_id` | `string` | Yes | UUID v4 format | Unique identifier for this specific message | | `msg_id` | `string` | Yes | UUID v4 format | Unique identifier for this specific message | FR-012, NFR-401 |
| `timestamp` | `string` | Yes | ISO 8601 UTC | Message publication timestamp (e.g., `2026-03-13T07:02:50.443Z`) | | `timestamp` | `string` | Yes | ISO 8601 UTC | Message publication timestamp | FR-012, NFR-401 |
| `send_to` | `string` | Yes | Non-empty string | NATS subject/topic to publish the message to | | `send_to` | `string` | Yes | Non-empty string | NATS subject/topic to publish the message to | FR-013 |
| `msg_purpose` | `string` | Yes | Enum | Purpose of the message (see `msg_purpose` enum) | | `msg_purpose` | `string` | Yes | Enum | Purpose of the message | FR-013 |
| `sender_name` | `string` | Yes | Non-empty string | Name of the sender application | | `sender_name` | `string` | Yes | Non-empty string | Name of the sender application | FR-013 |
| `sender_id` | `string` | Yes | UUID v4 format | Unique identifier for the sender | | `sender_id` | `string` | Yes | UUID v4 format | Unique identifier for the sender | FR-013 |
| `receiver_name` | `string` | Yes | Any string | Name of the receiver (empty = broadcast) | | `receiver_name` | `string` | Yes | Any string | Name of the receiver (empty = broadcast) | FR-013 |
| `receiver_id` | `string` | Yes | Any string | UUID of the receiver (empty = broadcast) | | `receiver_id` | `string` | Yes | Any string | UUID of the receiver (empty = broadcast) | FR-013 |
| `reply_to` | `string` | Yes | Any string | Topic where receiver should reply (empty = no reply expected) | | `reply_to` | `string` | Yes | Any string | Topic where receiver should reply | FR-013 |
| `reply_to_msg_id` | `string` | Yes | Any string | Message ID this message is replying to | | `reply_to_msg_id` | `string` | Yes | Any string | Message ID this message is replying to | FR-013 |
| `broker_url` | `string` | Yes | Valid URL | NATS broker URL | | `broker_url` | `string` | Yes | Valid URL | NATS broker URL | FR-013 |
| `metadata` | `object` | No | Any JSON object | Message-level metadata | | `metadata` | `object` | No | Any JSON object | Message-level metadata | NFR-401 |
| `payloads` | `array` | Yes | Non-empty array | List of payload objects | | `payloads` | `array` | Yes | Non-empty array | List of payload objects | FR-012, FR-013 |
--- ---
@@ -103,16 +128,16 @@ This specification serves as the single source of truth for:
### Payload Field Definitions ### Payload Field Definitions
| Field | Type | Required | Validation | Description | | Field | Type | Required | Validation | Description | Requirement ID |
|-------|------|----------|------------|-------------| |-------|------|----------|------------|-------------|----------------|
| `id` | `string` | Yes | UUID v4 format | Unique identifier for this payload | | `id` | `string` | Yes | UUID v4 format | Unique identifier for this payload | FR-012 |
| `dataname` | `string` | Yes | Non-empty string | Name of the payload (e.g., `login_image`, `user_data`) | | `dataname` | `string` | Yes | Non-empty string | Name of the payload (e.g., `login_image`, `user_data`) | FR-001, FR-002, FR-003 |
| `payload_type` | `string` | Yes | Enum | Type of payload (see `payload_type` enum) | | `payload_type` | `string` | Yes | Enum | Type of payload (see `payload_type` enum) | FR-001, FR-002, FR-003, FR-006 |
| `transport` | `string` | Yes | Enum | Transport method: `direct` or `link` | | `transport` | `string` | Yes | Enum | Transport method: `direct` or `link` | FR-003, FR-004, NFR-104, NFR-105 |
| `encoding` | `string` | Yes | Enum | Encoding method (see `encoding` enum) | | `encoding` | `string` | Yes | Enum | Encoding method (see `encoding` enum) | FR-001, FR-002, FR-003, FR-012 |
| `size` | `integer` | Yes | Positive integer | Size of the payload in bytes | | `size` | `integer` | Yes | Positive integer | Size of the payload in bytes | FR-003, FR-004, NFR-104, NFR-105 |
| `data` | `string` or `URL` | Yes | Base64 string or URL | Payload data (base64 for direct, URL for link) | | `data` | `string` or `URL` | Yes | Base64 string or URL | Payload data (base64 for direct, URL for link) | FR-003, FR-004, FR-008, FR-009, FR-010 |
| `metadata` | `object` | No | Any JSON object | Payload-level metadata | | `metadata` | `object` | No | Any JSON object | Payload-level metadata | NFR-401 |
--- ---
@@ -176,6 +201,7 @@ await smartsend("/agent/v1/process", data)
| All | `String` | `"text"` | | All | `String` | `"text"` |
| All | `Dict`/`Object` | `"dictionary"` | | All | `Dict`/`Object` | `"dictionary"` |
| Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` | | Desktop | `DataFrame` | `"arrowtable"` or `"jsontable"` |
| Browser | `Array` of objects | `"jsontable"` (only table type) |
| All | `Array` of objects | `"jsontable"` | | All | `Array` of objects | `"jsontable"` |
| All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` | | All | `Uint8Array`/`Buffer`/`bytes` | `"binary"` |
| Desktop | `Arrow.Table` | `"arrowtable"` | | Desktop | `Arrow.Table` | `"arrowtable"` |
@@ -203,8 +229,8 @@ await smartsend("/agent/v1/process", data)
|-------|-------------|---------------------|------------------| |-------|-------------|---------------------|------------------|
| `text` | Plain text string | All | `base64` | | `text` | Plain text string | All | `base64` |
| `dictionary` | JSON object/dictionary | All | `base64`, `json` | | `dictionary` | JSON object/dictionary | All | `base64`, `json` |
| `arrowtable` | Apache Arrow IPC table | Desktop (Julia/JS/Python) | `base64`, `arrow-ipc` | | `arrowtable` | Apache Arrow IPC table | Desktop (Julia/Python/Node.js/Dart) | `base64`, `arrow-ipc` |
| `jsontable` | JSON array of objects | All | `base64`, `json` | | `jsontable` | JSON array of objects | All (including Browser/Dart Web) | `base64`, `json` |
| `image` | Binary image data | All | `base64` | | `image` | Binary image data | All | `base64` |
| `audio` | Binary audio data | All | `base64` | | `audio` | Binary audio data | All | `base64` |
| `video` | Binary video data | All | `base64` | | `video` | Binary video data | All | `base64` |
@@ -324,25 +350,25 @@ When `transport = "link"`, the `data` field contains a URL pointing to the uploa
### Error Codes ### Error Codes
| Code | HTTP Status | Description | Recovery | | Code | HTTP Status | Description | Recovery | Requirement ID |
|------|-------------|-------------|----------| |------|-------------|-------------|----------|----------------|
| `INVALID_ENVELOPE` | 400 | Message envelope validation failed | Fix envelope structure | | `INVALID_ENVELOPE` | 400 | Message envelope validation failed | Fix envelope structure | FR-012, FR-013, FR-014 |
| `INVALID_PAYLOAD_TYPE` | 400 | Unsupported payload type | Use supported payload_type | | `INVALID_PAYLOAD_TYPE` | 400 | Unsupported payload type | Use supported payload_type | FR-001, FR-002, FR-003, FR-006 |
| `INVALID_TRANSPORT` | 400 | Unsupported transport type | Use `direct` or `link` | | `INVALID_TRANSPORT` | 400 | Unsupported transport type | Use `direct` or `link` | FR-003, FR-004, FR-006 |
| `UPLOAD_FAILED` | 500 | File server upload failed | Retry or use direct transport | | `UPLOAD_FAILED` | 500 | File server upload failed | Retry or use direct transport | FR-008, FR-009 |
| `DOWNLOAD_FAILED` | 503 | File server download failed | Retry with exponential backoff | | `DOWNLOAD_FAILED` | 503 | File server download failed | Retry with exponential backoff | FR-010, FR-011, NFR-201, NFR-202 |
| `NATS_CONNECTION_FAILED` | 503 | NATS connection failed | Check NATS server availability | | `NATS_CONNECTION_FAILED` | 503 | NATS connection failed | Check NATS server availability | FR-013, FR-014, NFR-201, NFR-203 |
| `DESERIALIZATION_ERROR` | 500 | Payload deserialization failed | Check payload_type matches data | | `DESERIALIZATION_ERROR` | 500 | Payload deserialization failed | Check payload_type matches data | FR-001, FR-002, FR-003, FR-012 |
| `SIZE_EXCEEDED` | 413 | Payload exceeds maximum size | Split payload or use link transport | | `SIZE_EXCEEDED` | 413 | Payload exceeds maximum size | Split payload or use link transport | FR-003, FR-004, FR-005, NFR-104, NFR-105 |
### Exception Handling ### Exception Handling
| Scenario | Handler | Retry Policy | | Scenario | Handler | Retry Policy | Requirement ID |
|----------|---------|--------------| |----------|---------|--------------|----------------|
| File server unavailable | Retry up to 5 times | Exponential backoff (100ms → 5000ms) | | File server unavailable | Retry up to 5 times | Exponential backoff (100ms → 5000ms) | FR-010, NFR-202 |
| NATS publish failure | Connection auto-reconnect | TCP-level reconnection | | NATS publish failure | Connection auto-reconnect | TCP-level reconnection | FR-013, FR-014, NFR-201, NFR-203 |
| Deserialization error | Log correlation ID and throw | No retry (data corruption) | | Deserialization error | Log correlation ID and throw | No retry (data corruption) | FR-001, FR-002, FR-003, FR-012, NFR-401 |
| Memory overflow (MicroPython) | Reject payloads >50KB | No retry (client-side check) | | Memory overflow (MicroPython) | Reject payloads >50KB | No retry (client-side check) | FR-005, NFR-106 |
--- ---
@@ -480,11 +506,38 @@ async function smartsend(
reply_to?: string; reply_to?: string;
reply_to_msg_id?: string; reply_to_msg_id?: string;
is_publish?: boolean; is_publish?: boolean;
nats_connection?: NATS.Connection; nats_connection?: NATSClient | NATS.Connection;
msg_id?: string; msg_id?: string;
sender_id?: string; sender_id?: string;
} }
): Promise<[Object, string]>; ): Promise<[Object, string]>;
// NATSClient class for connection management
class NATSClient {
constructor(url: string, keepAlive?: boolean);
connect(): Promise<NATS.Connection>;
publish(subject: string, message: string, correlationId: string): Promise<void>;
close(): Promise<void>;
getConnection(): NATS.Connection | null;
isConnected(): boolean;
}
// NATSConnectionPool for managing multiple connections
class NATSConnectionPool {
constructor(url: string, maxSize?: number);
acquire(): Promise<NATSClient>;
release(client: NATSClient): void;
closeAll(): Promise<void>;
}
// publishMessage function for manual publishing
async function publishMessage(
brokerUrlOrClient: string | NATSClient | NATS.Connection,
subject: string,
message: string,
correlationId: string,
closeConnection?: boolean
): Promise<void>;
``` ```
#### MicroPython #### MicroPython
@@ -497,6 +550,58 @@ def smartsend(
) -> Tuple[Dict, str]: ) -> Tuple[Dict, str]:
``` ```
#### Dart (Desktop/Flutter)
```dart
Future<[Map<String, dynamic>, String]> smartsend(
String subject,
List<List<dynamic>> data, {
String brokerUrl = 'nats://localhost:4222',
String fileserverUrl = 'http://localhost:8080',
Function? fileserverUploadHandler,
int sizeThreshold = 500000,
String? correlationId,
String msgPurpose = 'chat',
String senderName = 'NATSBridge',
String receiverName = '',
String receiverId = '',
String replyTo = '',
String replyToMsgId = '',
bool isPublish = true,
dynamic natsConnection,
String? msgId,
String? senderId,
}) async {
// Returns [envelope, jsonString]
}
```
#### Dart Web
```dart
Future<[Map<String, dynamic>, String]> smartsend(
String subject,
List<List<dynamic>> data, {
String brokerUrl = 'nats://localhost:4222',
String fileserverUrl = 'http://localhost:8080',
Function? fileserverUploadHandler,
int sizeThreshold = 500000,
String? correlationId,
String msgPurpose = 'chat',
String senderName = 'NATSBridge',
String receiverName = '',
String receiverId = '',
String replyTo = '',
String replyToMsgId = '',
bool isPublish = true,
dynamic natsConnection,
String? msgId,
String? senderId,
}) async {
// Returns [envelope, jsonString]
}
```
### `smartreceive` Function Signature ### `smartreceive` Function Signature
#### Julia #### Julia
@@ -557,6 +662,34 @@ async function smartreceive(
def smartreceive(msg: Any, **kwargs) -> Dict[str, Any]: def smartreceive(msg: Any, **kwargs) -> Dict[str, Any]:
``` ```
#### Dart (Desktop/Flutter)
```dart
Future<Map<String, dynamic>> smartreceive(
Map<String, dynamic> msg, {
Function? fileserverDownloadHandler,
int maxRetries = 5,
int baseDelay = 100,
int maxDelay = 5000,
}) async {
// Returns envelope with processed payloads
}
```
#### Dart Web
```dart
Future<Map<String, dynamic>> smartreceive(
Map<String, dynamic> msg, {
Function? fileserverDownloadHandler,
int maxRetries = 5,
int baseDelay = 100,
int maxDelay = 5000,
}) async {
// Returns envelope with processed payloads
}
```
--- ---
## File Server Interface ## File Server Interface
@@ -613,16 +746,56 @@ function fileserver_download_handler(
## Platform-Specific Constraints ## Platform-Specific Constraints
### Desktop (Julia/JS/Python) ### Desktop (Julia/Python/Node.js/Dart)
| Feature | Status | Notes | | Feature | Status | Notes |
|---------|--------|-------| |---------|--------|-------|
| Arrow IPC | ✅ Supported | Requires Arrow.jl/pyarrow | | Arrow IPC | ✅ Supported | Requires Arrow.jl/pyarrow/dart-arrow |
| JSON table | ✅ Supported | Human-readable format | | JSON table | ✅ Supported | Human-readable format |
| File server upload | ✅ Supported | HTTP/HTTPS | | File server upload | ✅ Supported | HTTP/HTTPS |
| File server download | ✅ Supported | HTTP/HTTPS | | File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable | | Size threshold | 500KB | Configurable |
### Browser (JavaScript)
| Feature | Status | Notes |
|---------|--------|-------|
| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible |
| JSON table | ✅ Supported | Only table type available in browser |
| File server upload | ✅ Supported | HTTP/HTTPS |
| File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable |
### Dart Desktop (Dart SDK)
| Feature | Status | Notes |
|---------|--------|-------|
| Arrow IPC | ✅ Supported | Requires dart-arrow package |
| JSON table | ✅ Supported | Human-readable format |
| File server upload | ✅ Supported | HTTP/HTTPS |
| File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable |
### Dart Flutter (Dart SDK)
| Feature | Status | Notes |
|---------|--------|-------|
| Arrow IPC | ✅ Supported | Requires dart-arrow package |
| JSON table | ✅ Supported | Human-readable format |
| File server upload | ✅ Supported | HTTP/HTTPS |
| File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable |
### Dart Web (Dart SDK)
| Feature | Status | Notes |
|---------|--------|-------|
| Arrow IPC | ❌ Not supported | Apache Arrow not browser-compatible |
| JSON table | ✅ Supported | Only table type available in browser |
| File server upload | ✅ Supported | HTTP/HTTPS |
| File server download | ✅ Supported | HTTP/HTTPS |
| Size threshold | 500KB | Configurable |
### MicroPython ### MicroPython
| Feature | Status | Notes | | Feature | Status | Notes |
@@ -636,6 +809,42 @@ function fileserver_download_handler(
--- ---
## Implementation Files
| File | Platform | Features | Notes |
|------|----------|----------|-------|
| [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | Ground truth implementation |
| [`src/natsbridge_ssr.js`](../src/natsbridge_ssr.js) | Node.js | Arrow IPC, async/await | Server-side JavaScript |
| [`src/natsbridge_csr.js`](../src/natsbridge_csr.js) | Browser | JSON table only, WebSocket NATS | Client-side rendering |
| [`src/natsbridge.py`](../src/natsbridge.py) | Python | Arrow IPC, async/await | Desktop Python |
| [`src/natsbridge.dart`](../src/natsbridge.dart) | Dart | Full feature set, Arrow IPC, async/await | Desktop/Flutter/Web |
| [`src/natsbridge_mpy.py`](../src/natsbridge_mpy.py) | MicroPython | Limited to direct transport | Memory-constrained |
### Browser Implementation Notes
The browser implementation ([`src/natsbridge_csr.js`](../src/natsbridge_csr.js)) has the following constraints:
| Constraint | Reason | Workaround |
|------------|--------|------------|
| No Apache Arrow IPC | Browser-incompatible dependency | Use `jsontable` for tabular data |
| WebSocket NATS only | Browser cannot use TCP directly | Use `ws://` or `wss://` broker URLs |
| Fetch API for HTTP | Browser fetch() API only | Compatible with Plik and other HTTP servers |
### Payload Type Availability by Platform
| Payload Type | Julia | Node.js | Browser | Python | Dart | MicroPython |
|--------------|-------|---------|---------|--------|------|-------------|
| `text` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| `dictionary` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| `arrowtable` | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ |
| `jsontable` | ✅ | ✅ | ✅ | ✅ | ✅ | ⚠️ |
| `image` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| `audio` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| `video` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| `binary` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
---
## Message Flow ## Message Flow
### Sending Flow ### Sending Flow
@@ -686,23 +895,23 @@ flowchart TD
### Envelope Validation ### Envelope Validation
| Rule | Condition | Error Code | | Rule | Condition | Error Code | Requirement ID |
|------|-----------|------------| |------|-----------|------------|----------------|
| Required fields present | `correlation_id`, `msg_id`, `timestamp`, `send_to`, `payloads` | `INVALID_ENVELOPE` | | Required fields present | `correlation_id`, `msg_id`, `timestamp`, `send_to`, `payloads` | `INVALID_ENVELOPE` | FR-012, FR-013 |
| Valid UUID format | `correlation_id`, `msg_id`, `sender_id`, `receiver_id` | `INVALID_ENVELOPE` | | Valid UUID format | `correlation_id`, `msg_id`, `sender_id`, `receiver_id` | `INVALID_ENVELOPE` | FR-011, FR-012, NFR-401 |
| Valid timestamp format | ISO 8601 UTC | `INVALID_ENVELOPE` | | Valid timestamp format | ISO 8601 UTC | `INVALID_ENVELOPE` | FR-012, NFR-401 |
| Non-empty payloads array | `length(payloads) > 0` | `INVALID_ENVELOPE` | | Non-empty payloads array | `length(payloads) > 0` | `INVALID_ENVELOPE` | FR-012, FR-013 |
### Payload Validation ### Payload Validation
| Rule | Condition | Error Code | | Rule | Condition | Error Code | Requirement ID |
|------|-----------|------------| |------|-----------|------------|----------------|
| Valid payload_type | Must be in `payload_type` enum | `INVALID_PAYLOAD_TYPE` | | Valid payload_type | Must be in `payload_type` enum | `INVALID_PAYLOAD_TYPE` | FR-001, FR-002, FR-003, FR-006 |
| Valid transport | Must be `direct` or `link` | `INVALID_TRANSPORT` | | Valid transport | Must be `direct` or `link` | `INVALID_TRANSPORT` | FR-003, FR-004, FR-006 |
| Valid encoding | Must match payload_type and transport | `INVALID_TRANSPORT` | | Valid encoding | Must match payload_type and transport | `INVALID_TRANSPORT` | FR-001, FR-002, FR-003, FR-012 |
| Positive size | `size > 0` | `INVALID_PAYLOAD` | | Positive size | `size > 0` | `INVALID_PAYLOAD` | FR-003, FR-004, NFR-104, NFR-105 |
| Valid Base64 for direct | `data` matches Base64 pattern | `DESERIALIZATION_ERROR` | | Valid Base64 for direct | `data` matches Base64 pattern | `DESERIALIZATION_ERROR` | FR-001, FR-002, FR-003, FR-012 |
| Valid URL for link | `data` matches HTTP(S) URL pattern | `DOWNLOAD_FAILED` | | Valid URL for link | `data` matches HTTP(S) URL pattern | `DOWNLOAD_FAILED` | FR-008, FR-009, FR-010 |
--- ---
@@ -710,14 +919,14 @@ flowchart TD
### Unit Test Validation ### Unit Test Validation
| Test | Input | Expected Output | Notes | | Test | Input | Expected Output | Notes | Requirement ID |
|------|-------|-----------------|-------| |------|-------|-----------------|-------|----------------|
| Text round-trip | `("msg", "Hello", "text")` | `("msg", "Hello", "text")` | String serialization | | Text round-trip | `("msg", "Hello", "text")` | `("msg", "Hello", "text")` | String serialization | FR-001, FR-012, NFR-101, NFR-102 |
| Dictionary round-trip | `("data", {"key": "value"}, "dictionary")` | `("data", {"key": "value"}, "dictionary")` | JSON object round-trip | | Dictionary round-trip | `("data", {"key": "value"}, "dictionary")` | `("data", {"key": "value"}, "dictionary")` | JSON object round-trip | FR-002, FR-012, NFR-101, NFR-102 |
| Arrow table round-trip | `("table", arrow_table_data, "arrowtable")` | `("table", arrow_table_data, "arrowtable")` | Arrow IPC round-trip | | Arrow table round-trip | `("table", arrow_table_data, "arrowtable")` | `("table", arrow_table_data, "arrowtable")` | Arrow IPC round-trip | FR-002, FR-012, NFR-101, NFR-102 |
| JSON table round-trip | `("table", [{"a":1},{"b":2}], "jsontable")` | `("table", [{"a":1},{"b":2}], "jsontable")` | JSON array of objects | | JSON table round-trip | `("table", [{"a":1},{"b":2}], "jsontable")` | `("table", [{"a":1},{"b":2}], "jsontable")` | JSON array of objects | FR-001, FR-002, FR-006, FR-012 |
| Mixed payloads | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | Multiple payload types | | Mixed payloads | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | `[("msg", "Hello", "text"), ("imgname", bytes, "binary")]` | Multiple payload types | FR-006, FR-007 |
| Large payload | `("data", rand(10_000_000), "arrowtable")` | `("data", URL, "arrowtable")` with link transport | File server upload | | Large payload | `("data", rand(10_000_000), "arrowtable")` | `("data", URL, "arrowtable")` with link transport | File server upload | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 |
**Platform-Specific Notes:** **Platform-Specific Notes:**
- **Julia**: Use `Dict`, `Vector{Dict}`, or convert `DataFrame` to dictionary for testing - **Julia**: Use `Dict`, `Vector{Dict}`, or convert `DataFrame` to dictionary for testing
@@ -727,24 +936,24 @@ flowchart TD
### Integration Test Scenarios ### Integration Test Scenarios
| Scenario | Platforms | Payloads | Size Mix | Transport | Expected Result | | Scenario | Platforms | Payloads | Size Mix | Transport | Expected Result | Requirement ID |
|----------|-----------|----------|----------|-----------|-----------------| |----------|-----------|----------|----------|-----------|-----------------|----------------|
| Single text (small) | All | `text` | Small | direct | Round-trip successful | | Single text (small) | All | `text` | Small | direct | Round-trip successful | FR-001, FR-012, NFR-101, NFR-102 |
| Single dictionary (small) | All | `dictionary` | Small | direct | Round-trip successful | | Single dictionary (small) | All | `dictionary` | Small | direct | Round-trip successful | FR-002, FR-012, NFR-101, NFR-102 |
| Single arrow table (small) | Julia/JS/Python | `arrowtable` | Small | direct | Arrow IPC round-trip | | Single arrow table (small) | Julia/JS/Python | `arrowtable` | Small | direct | Arrow IPC round-trip | FR-002, FR-012, NFR-101, NFR-102 |
| Single JSON table (small) | All | `jsontable` | Small | direct | Dictionary array round-trip | | Single JSON table (small) | All | `jsontable` | Small | direct | Dictionary array round-trip | FR-001, FR-002, FR-006, FR-012 |
| Single image (small) | All | `image` | Small | direct | Binary round-trip | | Single image (small) | All | `image` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 |
| Single audio (small) | All | `audio` | Small | direct | Binary round-trip | | Single audio (small) | All | `audio` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 |
| Single video (small) | All | `video` | Small | direct | Binary round-trip | | Single video (small) | All | `video` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 |
| Single binary (small) | All | `binary` | Small | direct | Binary round-trip | | Single binary (small) | All | `binary` | Small | direct | Binary round-trip | FR-001, FR-006, FR-012 |
| Single text (large) | All | `text` | Large | link | File server upload/download | | Single text (large) | All | `text` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 |
| Single JSON table (large) | All | `jsontable` | Large | link | File server upload/download | | Single JSON table (large) | All | `jsontable` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 |
| Single image (large) | All | `image` | Large | link | File server upload/download | | Single image (large) | All | `image` | Large | link | File server upload/download | FR-003, FR-004, FR-008, FR-009, NFR-104, NFR-105 |
| **Ultimate Test** | Julia/JS/Python | `text` (small) + `dictionary` (small) + `arrowtable` (small) + `jsontable` (small) + `image` (small) + `audio` (small) + `video` (small) + `binary` (small) + `text` (large) + `dictionary` (large) + `arrowtable` (large) + `jsontable` (large) + `image` (large) | Mixed | direct/link | All payloads preserved with correct transport | | **Ultimate Test** | Julia/JS/Python | `text` (small) + `dictionary` (small) + `arrowtable` (small) + `jsontable` (small) + `image` (small) + `audio` (small) + `video` (small) + `binary` (small) + `text` (large) + `dictionary` (large) + `arrowtable` (large) + `jsontable` (large) + `image` (large) | Mixed | direct/link | All payloads preserved with correct transport | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-008, FR-009, FR-010, FR-011, FR-012, FR-013, FR-014 |
| **Ultimate Test** | MicroPython | `text` (small) + `dictionary` (small) + `text` (large) + `dictionary` (large) | Mixed | direct | Limited to text/dictionary with direct transport only | | **Ultimate Test** | MicroPython | `text` (small) + `dictionary` (small) + `text` (large) + `dictionary` (large) | Mixed | direct | Limited to text/dictionary with direct transport only | FR-005, FR-006, FR-012 |
| Cross-platform JSON table | All | `jsontable` | Small | direct | Dictionary array round-trip | | Cross-platform JSON table | All | `jsontable` | Small | direct | Dictionary array round-trip | FR-001, FR-002, FR-006, FR-012 |
| MicroPython ↔ Desktop | MicroPython ↔ Desktop | `text`/`dictionary` | Small | direct | Limited payload types | | MicroPython ↔ Desktop | MicroPython ↔ Desktop | `text`/`dictionary` | Small | direct | Limited payload types | FR-005, FR-006, FR-012 |
| Desktop ↔ Desktop (all combos) | Julia↔JS↔Python | All types | Small/Large | direct/link | Full compatibility | | Desktop ↔ Desktop (all combos) | Julia↔JS↔Python | All types | Small/Large | direct/link | Full compatibility | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 |
--- ---
@@ -759,11 +968,17 @@ flowchart TD
| Julia | Arrow.jl | Latest | Arrow IPC support | | Julia | Arrow.jl | Latest | Arrow IPC support |
| Julia | HTTP.jl | Latest | HTTP file server | | Julia | HTTP.jl | Latest | HTTP file server |
| Julia | UUIDs.jl | Latest | UUID generation | | Julia | UUIDs.jl | Latest | UUID generation |
| Node.js | nats | Latest | NATS client | | Node.js | nats | Latest | NATS client (TCP) |
| Node.js | node-fetch | Latest | HTTP file server | | Node.js | node-fetch | Latest | HTTP file server |
| Browser | nats.ws | Latest | NATS client (WebSocket) |
| Browser | nats | Latest | NATS client (for bundling) |
| Python | nats-py | Latest | NATS client | | Python | nats-py | Latest | NATS client |
| Python | aiohttp | Latest | HTTP file server | | Python | aiohttp | Latest | HTTP file server |
| Python | pyarrow | Latest | Arrow IPC support | | Python | pyarrow | Latest | Arrow IPC support |
| Dart | nats | Latest | NATS client |
| Dart | http | Latest | HTTP file server |
| Dart | uuid | Latest | UUID generation |
| Dart | dart-arrow | Latest | Arrow IPC support (Desktop/Flutter) |
| MicroPython | builtin | N/A | Limited implementation | | MicroPython | builtin | N/A | Limited implementation |
### Optional Dependencies ### Optional Dependencies
@@ -779,6 +994,11 @@ flowchart TD
| Date | Version | Changes | | Date | Version | Changes |
|------|---------|---------| |------|---------|---------|
| 2026-03-15 | 1.1.0 | Browser connection management |
| - | - | Added NATSClient class with keepAlive support |
| - | - | Added NATSConnectionPool for connection reuse |
| - | - | Added publishMessage function with closeConnection option |
| - | - | Added nats.ws to browser dependencies |
| 2026-03-13 | 1.0.0 | Initial specification | | 2026-03-13 | 1.0.0 | Initial specification |
| - | - | Message envelope schema defined | | - | - | Message envelope schema defined |
| - | - | Payload schema with transport modes | | - | - | Payload schema with transport modes |
@@ -791,11 +1011,55 @@ flowchart TD
## References ## References
- [`docs/requirements.md`](./requirements.md) - Business requirements and user stories ### 20.1 Documentation Artifacts
- [`docs/architecture.md`](./architecture.md) - System architecture diagrams
- [`docs/implementation.md`](./implementation.md) - Implementation details | Document | Purpose | Requirements Traceability |
- [`src/NATSBridge.jl`](../src/NATSBridge.jl) - Ground truth implementation |----------|---------|--------------------------|
- [`README.md`](../README.md) - Project overview | [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/specification.md`](./specification.md) | Technical contract for NATSBridge | This document |
| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | UI components for data entry and display |
| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | Traceability from user journey to technical implementation |
| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | Component interaction and data flow |
| [`docs/validation.md`](./validation.md) | CI/CD validation rules | Contract testing and spec compliance |
| [`docs/runbook.md`](./runbook.md) | Operational runbook | Deployment, scaling, and troubleshooting |
### 20.2 Implementation Files
| File | Platform | Features | Requirements Traceability |
|------|----------|----------|--------------------------|
| [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge_ssr.js`](../src/natsbridge_ssr.js) | Node.js | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge_csr.js`](../src/natsbridge_csr.js) | Browser | JSON table only, WebSocket NATS | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge.py`](../src/natsbridge.py) | Python | Arrow IPC, async/await | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`src/natsbridge_mpy.py`](../src/natsbridge_mpy.py) | MicroPython | Limited to direct transport | FR-005, FR-006, FR-012 |
### 20.3 External Dependencies
| Platform | Package | Version | Purpose | Requirements Traceability |
|----------|---------|---------|---------|--------------------------|
| Julia | NATS.jl | Latest | NATS client | FR-013, FR-014, NFR-201 |
| Julia | JSON.jl | Latest | JSON serialization | FR-012, NFR-101, NFR-102 |
| Julia | Arrow.jl | Latest | Arrow IPC support | FR-002, FR-012 |
| Julia | HTTP.jl | Latest | HTTP file server | FR-008, FR-009 |
| Julia | UUIDs.jl | Latest | UUID generation | FR-011, NFR-401 |
| Node.js | nats | Latest | NATS client (TCP) | FR-013, FR-014 |
| Node.js | node-fetch | Latest | HTTP file server | FR-008, FR-009 |
| Browser | nats.ws | Latest | NATS client (WebSocket) | FR-013, FR-014 |
| Browser | nats | Latest | NATS client (for bundling) | FR-013, FR-014 |
| Python | nats-py | Latest | NATS client | FR-013, FR-014 |
| Python | aiohttp | Latest | HTTP file server | FR-008, FR-009 |
| Python | pyarrow | Latest | Arrow IPC support | FR-002, FR-012 |
| MicroPython | builtin | N/A | Limited implementation | FR-005, FR-006 |
---
## 21. Change Log
| Date | Version | Changes | Requirement ID(s) |
|------|---------|---------|-------------------|
| 2026-03-23 | 1.1.0 | Updated to ASG Framework specification guidelines | All |
| 2026-03-15 | 1.1.0 | Browser connection management | FR-001 through FR-014 |
| 2026-03-13 | 1.0.0 | Initial specification | FR-001 through FR-014, NFR-101 through NFR-405 |
--- ---

View File

@@ -1,741 +0,0 @@
# Cross-Platform NATSBridge Tutorial
A step-by-step guide to get started with NATSBridge across **Julia**, **JavaScript**, and **Python/MicroPython**.
## Table of Contents
1. [Overview](#overview)
2. [Prerequisites](#prerequisites)
3. [Installation](#installation)
4. [Quick Start](#quick-start)
5. [Basic Examples](#basic-examples)
6. [Advanced Usage](#advanced-usage)
---
## Overview
NATSBridge enables seamless communication across platforms through NATS, with automatic transport selection based on payload size:
- **Direct Transport**: Payloads < 1MB are sent directly via NATS (Base64 encoded)
- **Link Transport**: Payloads >= 1MB are uploaded to an HTTP file server and referenced via URL
### Cross-Platform API Parity
All three platforms use the same high-level API:
```
# Input format
smartsend(subject, [(dataname, data, type), ...], options)
# Output format
(env, env_json_str) = smartsend(...)
env = smartreceive(msg, options)
```
**Important Platform Differences:**
1. **Encoding field:** Julia and JavaScript preserve the original serialization format in the encoding field (`"base64"`, `"json"`, or `"arrow-ipc"`), while Python and MicroPython always use `"base64"` for all direct transport payloads.
2. **Async vs Sync:** JavaScript and Python desktop use async/await, while MicroPython uses synchronous API.
### Supported Payload Types
| Type | Julia | JavaScript | Python | MicroPython |
|------|-------|------------|--------|-------------|
| `text` | `String` | `string` | `str` | `str` |
| `dictionary` | `Dict` | `Object` | `dict` | `dict` |
| `arrowtable` | `DataFrame` | `Array<Object>` | `pandas.DataFrame` | ❌ |
| `jsontable` | `Vector{NamedTuple}` | `Array<Object>` | `list[dict]` | ❌ |
| `table` | ❌ | ❌ | `pandas.DataFrame` | ❌ |
| `image` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` |
| `audio` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` |
| `video` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` |
| `binary` | `Vector{UInt8}` | `Uint8Array` | `bytes` | `bytearray` |
**Note on MicroPython:** MicroPython does not support table types (`arrowtable`, `jsontable`, or `table`) due to memory constraints. Use `dictionary` or `binary` instead.
---
## Prerequisites
Before you begin, ensure you have:
1. **NATS Server** running (or accessible)
2. **HTTP File Server** (optional, for large payloads > 1MB)
3. **Platform-specific packages** installed
---
## Installation
### Julia
```julia
using Pkg
Pkg.add("NATS")
Pkg.add("Arrow")
Pkg.add("JSON3")
Pkg.add("HTTP")
Pkg.add("UUIDs")
Pkg.add("Dates")
```
### JavaScript (Node.js)
```bash
npm install nats uuid apache-arrow node-fetch
```
### JavaScript (Browser)
```html
<script src="https://unpkg.com/nats-js/dist/bundle/nats.min.js"></script>
<script src="https://unpkg.com/apache-arrow/arrow.min.js"></script>
```
### Python (Desktop)
```bash
pip install nats-py aiohttp pyarrow pandas
```
### MicroPython
Uses built-in modules: `network`, `socket`, `time`, `json`, `base64`
---
## Quick Start
### Step 1: Start NATS Server
```bash
docker run -p 4222:4222 nats:latest
```
### Step 2: Start HTTP File Server (Optional)
```bash
mkdir -p /tmp/fileserver
python3 -m http.server 8080 --directory /tmp/fileserver
```
### Step 3: Send Your First Message
#### Julia
```julia
using NATSBridge
# Send a text message
data = [("message", "Hello World", "text")]
env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222")
# env: msg_envelope_v1 struct with all metadata and payloads
# env_json_str: JSON string representation of the envelope for publishing
println("Message sent!")
# Or use is_publish=false to get envelope and JSON without publishing
env, env_json_str = smartsend("/chat/room1", data, broker_url="nats://localhost:4222", is_publish=false)
# env: msg_envelope_v1 struct
# env_json_str: JSON string for publishing to NATS
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
// Send a text message
const data = [["message", "Hello World", "text"]];
const [env, env_json_str] = await NATSBridge.smartsend(
"/chat/room1",
data,
{ broker_url: "nats://localhost:4222" }
);
// env: Object with all metadata and payloads
// env_json_str: JSON string for publishing
console.log("Message sent!");
// Or use is_publish=false
const [env, env_json_str] = await NATSBridge.smartsend(
"/chat/room1",
data,
{ broker_url: "nats://localhost:4222", is_publish: false }
);
```
#### Python
```python
from natsbridge import smartsend
# Send a text message
data = [("message", "Hello World", "text")]
env, env_json_str = await smartsend(
"/chat/room1",
data,
broker_url="nats://localhost:4222"
)
# env: Dict with all metadata and payloads
# env_json_str: JSON string for publishing
print("Message sent!")
# Or use is_publish=False
env, env_json_str = await smartsend(
"/chat/room1",
data,
broker_url="nats://localhost:4222",
is_publish=False
)
# env: Dict with all metadata and payloads
# env_json_str: JSON string for publishing to NATS
```
#### MicroPython
```python
from natsbridge_mpy import NATSBridge
bridge = NATSBridge()
# Send a text message (limited to small payloads)
data = [("message", "Hello World", "text")]
env, env_json_str = bridge.smartsend(
"/chat/room1",
data,
size_threshold=100000 # Lower threshold for MicroPython
)
print("Message sent!")
```
### Step 4: Receive Messages
#### Julia
```julia
using NATSBridge
# Receive and process message
env = smartreceive(msg; fileserver_download_handler=_fetch_with_backoff)
# Returns: ::JSON.Object{String, Any} with "payloads" field containing Vector{Tuple{String, Any, String}}
# Access payloads: for (dataname, data, type) in env["payloads"]
for (dataname, data, type) in env["payloads"]
println("Received $dataname: $data")
end
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
// Receive and process message
const env = await NATSBridge.smartreceive(msg, {
fileserver_download_handler: NATSBridge.fetchWithBackoff
});
// env.payloads = [[dataname, data, type], ...]
for (const [dataname, data, type] of env.payloads) {
console.log(`Received ${dataname}:`, data);
}
```
#### Python
```python
from natsbridge import smartreceive, fetch_with_backoff
# Receive and process message
env = await smartreceive(
msg,
fileserver_download_handler=fetch_with_backoff
)
# env["payloads"] = [(dataname, data, type), ...]
for dataname, data, type_ in env["payloads"]:
print(f"Received {dataname}: {data}")
```
---
## Basic Examples
### Example 1: Sending a Dictionary
#### Julia
```julia
using NATSBridge
config = Dict(
"wifi_ssid" => "MyNetwork",
"wifi_password" => "password123",
"update_interval" => 60
)
data = [("config", config, "dictionary")]
env, env_json_str = smartsend("/device/config", data, broker_url="nats://localhost:4222")
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
const config = {
wifi_ssid: "MyNetwork",
wifi_password: "password123",
update_interval: 60
};
const data = [["config", config, "dictionary"]];
const [env, env_json_str] = await NATSBridge.smartsend(
"/device/config",
data,
{ broker_url: "nats://localhost:4222" }
);
```
#### Python
```python
from natsbridge import smartsend
config = {
"wifi_ssid": "MyNetwork",
"wifi_password": "password123",
"update_interval": 60
}
data = [("config", config, "dictionary")]
env, env_json_str = await smartsend(
"/device/config",
data,
broker_url="nats://localhost:4222"
)
```
#### MicroPython
```python
from natsbridge_mpy import NATSBridge
bridge = NATSBridge()
config = {
"wifi_ssid": "MyNetwork",
"wifi_password": "password123",
"update_interval": 60
}
data = [("config", config, "dictionary")]
env, env_json_str = bridge.smartsend(
"/device/config",
data,
size_threshold=100000
)
```
### Example 2: Sending Binary Data (Image)
#### Julia
```julia
using NATSBridge
# Read image file
image_data = read("image.png")
data = [("user_image", image_data, "binary")]
env, env_json_str = smartsend("/chat/image", data, broker_url="nats://localhost:4222")
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
const fs = require('fs');
// Read image file
const image_data = fs.readFileSync('image.png');
const data = [["user_image", image_data, "binary"]];
const [env, env_json_str] = await NATSBridge.smartsend(
"/chat/image",
data,
{ broker_url: "nats://localhost:4222" }
);
```
#### Python
```python
from natsbridge import smartsend
# Read image file
with open("image.png", "rb") as f:
image_data = f.read()
data = [("user_image", image_data, "binary")]
env, env_json_str = await smartsend(
"/chat/image",
data,
broker_url="nats://localhost:4222"
)
```
#### MicroPython
```python
from natsbridge_mpy import NATSBridge
bridge = NATSBridge()
# Read image file
with open("image.png", "rb") as f:
image_data = f.read()
data = [("user_image", image_data, "binary")]
env, env_json_str = bridge.smartsend(
"/chat/image",
data,
size_threshold=100000
)
```
### Example 3: Request-Response Pattern
#### Julia (Requester)
```julia
using NATSBridge
# Send command with reply-to
data = [("command", Dict("action" => "read_sensor"), "dictionary")]
env, env_json_str = smartsend(
"/device/command",
data,
broker_url="nats://localhost:4222",
reply_to="/device/response",
reply_to_msg_id="cmd-001"
)
```
#### JavaScript (Requester)
```javascript
const NATSBridge = require('./src/natsbridge.js');
// Send command with reply-to
const data = [["command", { action: "read_sensor" }, "dictionary"]];
const [env, env_json_str] = await NATSBridge.smartsend(
"/device/command",
data,
{
broker_url: "nats://localhost:4222",
reply_to: "/device/response",
reply_to_msg_id: "cmd-001"
}
);
```
#### Python (Requester)
```python
from natsbridge import smartsend
# Send command with reply-to
data = [("command", {"action": "read_sensor"}, "dictionary")]
env, env_json_str = await smartsend(
"/device/command",
data,
broker_url="nats://localhost:4222",
reply_to="/device/response",
reply_to_msg_id="cmd-001"
)
```
#### Julia (Responder)
```julia
using NATSBridge, NATS
const SUBJECT = "/device/command"
const NATS_URL = "nats://localhost:4222"
function test_responder()
conn = NATS.connect(NATS_URL)
NATS.subscribe(conn, SUBJECT) do msg
env = smartreceive(msg, fileserver_download_handler=_fetch_with_backoff)
reply_to = env["reply_to"]
for (dataname, data, type) in env["payloads"]
if dataname == "command" && data["action"] == "read_sensor"
response = Dict("sensor_id" => "sensor-001", "value" => 42.5)
if !isempty(reply_to)
smartsend(reply_to, [("data", response, "dictionary")])
end
end
end
end
sleep(120)
NATS.drain(conn)
end
test_responder()
```
---
## Advanced Usage
### Example 4: Large Payloads (File Server)
For payloads larger than 1MB, NATSBridge automatically uses the file server:
#### Julia
```julia
using NATSBridge
# Create large data (> 1MB)
large_data = rand(UInt8, 2_000_000)
env, env_json_str = smartsend(
"/data/large",
[("large_file", large_data, "binary")],
broker_url="nats://localhost:4222",
fileserver_url="http://localhost:8080"
)
println("File uploaded to: $(env.payloads[1].data)")
# Note: For link transport, data field contains the URL string
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
// Create large data (> 1MB)
const large_data = Buffer.alloc(2_000_000);
for (let i = 0; i < large_data.length; i++) {
large_data[i] = Math.floor(Math.random() * 256);
}
const [env, env_json_str] = await NATSBridge.smartsend(
"/data/large",
[["large_file", large_data, "binary"]],
{
broker_url: "nats://localhost:4222",
fileserver_url: "http://localhost:8080"
}
);
console.log("File uploaded to:", env.payloads[0].data);
// Note: For link transport, data field contains the URL string
```
#### Python
```python
from natsbridge import smartsend
# Create large data (> 1MB)
import os
large_data = os.urandom(2_000_000)
env, env_json_str = await smartsend(
"/data/large",
[("large_file", large_data, "binary")],
broker_url="nats://localhost:4222",
fileserver_url="http://localhost:8080"
)
print(f"File uploaded to: {env['payloads'][0]['data']}")
# Note: For link transport, data field contains the URL string
```
#### MicroPython
MicroPython enforces a hard limit of 50KB per payload:
```python
from natsbridge_mpy import NATSBridge
bridge = NATSBridge()
# MicroPython has a hard limit of 50KB per payload
# Use streaming or chunking for larger data
small_data = bytes(1000) # 1KB
data = [("small_file", small_data, "binary")]
env, env_json_str = bridge.smartsend(
"/data/small",
data,
size_threshold=100000 # Enforced max: 50000 bytes
)
```
### Example 5: Mixed Content (Chat with Text + Image)
NATSBridge supports sending multiple payloads with different types in a single message:
#### Julia
```julia
using NATSBridge
image_data = read("avatar.png")
data = [
("message_text", "Hello with image!", "text"),
("user_avatar", image_data, "image")
]
env, env_json_str = smartsend("/chat/mixed", data, broker_url="nats://localhost:4222")
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
const fs = require('fs');
const image_data = fs.readFileSync('avatar.png');
const data = [
["message_text", "Hello with image!", "text"],
["user_avatar", image_data, "image"]
];
const [env, env_json_str] = await NATSBridge.smartsend(
"/chat/mixed",
data,
{ broker_url: "nats://localhost:4222" }
);
```
#### Python
```python
from natsbridge import smartsend
with open("avatar.png", "rb") as f:
image_data = f.read()
data = [
("message_text", "Hello with image!", "text"),
("user_avatar", image_data, "image")
]
env, env_json_str = await smartsend(
"/chat/mixed",
data,
broker_url="nats://localhost:4222"
)
# env: Dict with all metadata and payloads
```
### Example 6: Table Data (Arrow IPC)
For tabular data, NATSBridge uses Apache Arrow IPC format:
#### Julia
```julia
using NATSBridge
using DataFrames
# Create DataFrame
df = DataFrame(
id = [1, 2, 3],
name = ["Alice", "Bob", "Charlie"],
score = [95, 88, 92]
)
data = [("students", df, "arrowtable")]
env, env_json_str = smartsend("/data/students", data, broker_url="nats://localhost:4222")
```
#### JavaScript
```javascript
const NATSBridge = require('./src/natsbridge.js');
// Create table data (array of objects)
const table_data = [
{ id: 1, name: "Alice", score: 95 },
{ id: 2, name: "Bob", score: 88 },
{ id: 3, name: "Charlie", score: 92 }
];
const data = [["students", table_data, "arrowtable"]];
const [env, env_json_str] = await NATSBridge.smartsend(
"/data/students",
data,
{ broker_url: "nats://localhost:4222" }
);
```
#### Python
```python
from natsbridge import smartsend
import pandas as pd
# Create DataFrame
df = pd.DataFrame({
'id': [1, 2, 3],
'name': ['Alice', 'Bob', 'Charlie'],
'score': [95, 88, 92]
})
data = [("students", df, "table")]
env, env_json_str = await smartsend(
"/data/students",
data,
broker_url="nats://localhost:4222"
)
```
#### MicroPython
MicroPython does not support table type due to memory constraints. Use dictionary or binary instead.
---
## Next Steps
1. **Explore the test directory** for more examples
2. **Check the documentation** for advanced configuration options
3. **Read the walkthrough** for building real-world applications
---
## Troubleshooting
### Connection Issues
- Ensure NATS server is running: `docker ps | grep nats`
- Check firewall settings
- Verify NATS URL configuration
### File Server Issues
- Ensure file server is running and accessible
- Check upload permissions
- Verify file server URL configuration
### Serialization Errors
- Verify data type matches the specified type
- Check that binary data is in the correct format
- MicroPython: Ensure payload size < 50KB
---
## License
MIT

View File

@@ -1,23 +1,40 @@
# Walkthrough: NATSBridge # Walkthrough: NATSBridge
**Version**: 1.0.0 **Version**: 1.0.0
**Date**: 2026-03-13 **Date**: 2026-03-23
**Status**: Active **Status**: Active
**Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl) **Ground Truth**: [`src/NATSBridge.jl`](../src/NATSBridge.jl)
--- ---
## Executive Summary ## 1. Executive Summary
This document provides the **story of flow** for NATSBridge - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, and **MicroPython** applications using NATS as the message bus. This document provides the **end-to-end trace** for NATSBridge - the cross-platform bi-directional data bridge that enables seamless communication between **Julia**, **JavaScript**, **Python**, **Dart**, and **MicroPython** applications using NATS as the message bus.
This walkthrough serves as the primary onboarding guide for new developers and explains: This walkthrough serves as the primary onboarding guide for new developers and explains:
- **User scenarios** - Real-world use cases from developer perspective - **User scenarios** - Real-world use cases from developer perspective
- **Why steps are sequenced** - The rationale behind architectural decisions - **Why steps are sequenced** - The rationale behind architectural decisions
- **What could go wrong** - Common failure scenarios and recovery strategies - **What could go wrong** - Common failure scenarios and recovery strategies
### 1.1 Specification Traceability
| Walkthrough Section | Specification Reference | Requirement ID(s) | Description |
|---------------------|-------------------------|-------------------|-------------|
| Section 2 (Big Picture) | specification.md:2, specification.md:15 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | End-to-end system flow diagrams |
| Section 3 (Chat Scenario) | specification.md:2, specification.md:3, specification.md:5, specification.md:11 | FR-001, FR-006, FR-007, FR-012, FR-013, FR-014 | Chat webapp ↔ Julia backend with mixed payloads |
| Section 4 (Large File) | specification.md:6, specification.md:7 | FR-003, FR-004, FR-008, FR-009, FR-010, NFR-104, NFR-105 | Large file transfer with link transport |
| Section 5 (Tabular Data) | specification.md:5, specification.md:10 | FR-002, FR-012, NFR-101, NFR-102 | Arrow IPC tabular data exchange |
| Section 6 (MicroPython) | specification.md:13, specification.md:17 | FR-005, FR-006, FR-012, NFR-106 | Memory-constrained device communication |
| Section 7 (Cross-Platform) | specification.md:3, specification.md:4, specification.md:5, specification.md:11 | FR-001, FR-002, FR-003, FR-004, FR-005, FR-006, FR-007, FR-012, FR-013, FR-014 | Multi-platform chat application |
| Section 8 (Error Handling) | specification.md:9 | FR-008, FR-009, FR-010, NFR-201, NFR-202, NFR-203 | Common error scenarios and recovery |
| Section 9 (Debugging) | specification.md:4, specification.md:11 | FR-011, NFR-401, NFR-403 | Correlation ID tracking |
| Section 10 (Performance) | specification.md:7, specification.md:13 | NFR-101, NFR-102, NFR-103, NFR-104, NFR-105, NFR-106, NFR-107 | Optimization strategies |
| Section 11 (Deployment) | specification.md:12, specification.md:18 | FR-013, FR-014, NFR-201, NFR-203 | Infrastructure requirements |
--- ---
## 2. Overview: The Big Picture
## Overview: The Big Picture ## Overview: The Big Picture
NATSBridge implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB): NATSBridge implements the **Claim-Check pattern** for efficient handling of large payloads (>0.5MB):
@@ -684,7 +701,10 @@ log_trace(correlation_id, "Published to NATS")
| Platform | Threshold | Notes | | Platform | Threshold | Notes |
|----------|-----------|-------| |----------|-----------|-------|
| Desktop (Julia/JS/Python) | 500,000 bytes (0.5MB) | Default threshold | | Desktop (Julia/JS/Python/Dart) | 500,000 bytes (0.5MB) | Default threshold |
| Dart Desktop | 500,000 bytes (0.5MB) | Default threshold |
| Dart Flutter | 500,000 bytes (0.5MB) | Default threshold |
| Dart Web | 500,000 bytes (0.5MB) | Default threshold |
| MicroPython | 100,000 bytes (100KB) | Lower threshold for memory constraints | | MicroPython | 100,000 bytes (100KB) | Lower threshold for memory constraints |
--- ---
@@ -697,7 +717,7 @@ log_trace(correlation_id, "Published to NATS")
|-----------|---------|-------| |-----------|---------|-------|
| NATS Server | 1 instance | Single node for development | | NATS Server | 1 instance | Single node for development |
| File Server | 1 instance | HTTP server for large payloads | | File Server | 1 instance | HTTP server for large payloads |
| Client Memory | 50MB | Desktop platforms | | Client Memory | 50MB | Desktop platforms (Julia/JS/Python/Dart) |
| Client Memory | 256KB | MicroPython devices | | Client Memory | 256KB | MicroPython devices |
### Environment Variables ### Environment Variables
@@ -718,13 +738,54 @@ log_trace(correlation_id, "Published to NATS")
--- ---
## References ## 12. References
- [`docs/requirements.md`](./requirements.md) - Business requirements and user stories ### 12.1 Documentation Artifacts
- [`docs/spec.md`](./spec.md) - Technical specification and contracts
- [`docs/architecture.md`](./architecture.md) - System architecture diagrams | Document | Purpose | Specification Traceability |
- [`src/NATSBridge.jl`](../src/NATSBridge.jl) - Ground truth implementation |----------|---------|---------------------------|
- [`README.md`](../README.md) - Project overview | [`docs/requirements.md`](./requirements.md) | Business requirements and user stories | FR-001 through FR-014, NFR-101 through NFR-405 |
| [`docs/specification.md`](./specification.md) | Technical contract for NATSBridge | specification.md:2-19 (all sections) |
| [`docs/ui-specification.md`](./ui-specification.md) | UI specification for client applications | UI components for data entry and display |
| [`docs/walkthrough.md`](./walkthrough.md) | End-to-end system flow | This document |
| [`docs/architecture.md`](./architecture.md) | System architecture diagrams | Component interaction and data flow |
| [`docs/validation.md`](./validation.md) | CI/CD validation rules | Contract testing and spec compliance |
| [`docs/runbook.md`](./runbook.md) | Operational runbook | Deployment, scaling, and troubleshooting |
### 12.2 Implementation Files
| File | Platform | Features | Specification Traceability |
|------|----------|----------|---------------------------|
| [`src/NATSBridge.jl`](../src/NATSBridge.jl) | Julia | Full feature set, Arrow IPC, multiple dispatch | specification.md:2-19 (all sections) |
| [`src/natsbridge_ssr.js`](../src/natsbridge_ssr.js) | Node.js | Arrow IPC, async/await | specification.md:2-19 (all sections) |
| [`src/natsbridge_csr.js`](../src/natsbridge_csr.js) | Browser | JSON table only, WebSocket NATS | specification.md:2-19 (all sections) |
| [`src/natsbridge.py`](../src/natsbridge.py) | Python | Arrow IPC, async/await | specification.md:2-19 (all sections) |
| [`src/natsbridge.dart`](../src/natsbridge.dart) | Dart | Full feature set, Arrow IPC, async/await | specification.md:2-19 (all sections) |
| [`src/natsbridge_mpy.py`](../src/natsbridge_mpy.py) | MicroPython | Limited to direct transport | specification.md:2-19 (all sections) |
---
## 13. Change Log
| Date | Version | Changes | Specification Reference |
|------|---------|---------|------------------------|
| 2026-03-23 | 1.0.0 | Updated to ASG Framework walkthrough guidelines | All sections |
| 2026-03-13 | 1.0.0 | Initial walkthrough documentation | specification.md:2-19 (all sections) |
---
## 14. Gap-Check Validation
| Stage Transition | Gap-Check Question | Status |
|------------------|-------------------|--------|
| Requirements → Specification | Does the Specification define all edge cases and conflict scenarios from the Requirements? | ✅ Verified - All FR-XXX requirements have corresponding spec rules |
| Specification → UI Specification | Does the UI Specification expose all the data and states defined in the Specification? | ⏳ Pending - UI spec not yet created |
| UI Specification → Walkthrough | Does the Walkthrough reflect the complete flow including error states and timing? | ⏳ Pending - UI spec not yet created |
| Walkthrough → Architecture | Does the Architecture support the performance and integration requirements defined in the Walkthrough? | ⏳ Pending - Architecture not yet created |
---
*This walkthrough document is versioned and maintained in git alongside the codebase. All implementations must adhere to this documentation.*
--- ---

56
etc.jl Normal file
View File

@@ -0,0 +1,56 @@
using HTTP, Arrow, JSON, DataFrames
df = DataFrame(id = 1:10_000, val = rand(10_000))
file_server_url = "http://192.168.88.104:8080"
url_getUploadID = "$file_server_url/api/upload"
function upload_to_plik(url, df)
# 1. Build the Request object manually
headers = [
"X-Plik-TTL" => "5m",
"Content-Type" => "application/octet-stream",
"Transfer-Encoding" => "chunked"
]
# We create a request with an empty body, but we'll stream into it
req = HTTP.Request("POST", url, headers)
# 2. Open the connection manually to get a raw Stream
local_url = ""
HTTP.open("POST", url, headers) do stream
# WRITE PHASE
# Arrow.write handles the 'chunked' encoding automatically
Arrow.write(stream, df; file=false)
# CLOSE WRITE / START READ
# This is the critical hand-off.
# We tell the kernel we are done sending.
HTTP.closewrite(stream)
# Now we wait for the server's response
resp = HTTP.startread(stream)
# Handle the body
if resp.status == 200 || resp.status == 201
payload = read(stream, String)
# Depending on Plik version, it might return the URL directly
# or a JSON object. Adjust accordingly:
try
local_url = JSON.parse(payload)["url"]
catch
local_url = payload # Fallback if it's a raw string
end
else
error("Plik rejected upload with status: $(resp.status)")
end
end
return local_url
end
url = upload_to_plik(url_getUploadID, df)

782
src/natsbridge.dart Normal file
View File

@@ -0,0 +1,782 @@
/// NATSBridge - Cross-Platform Bi-Directional Data Bridge
/// Dart Implementation (Desktop/Flutter/Web)
///
/// This module provides functionality for sending and receiving data across network boundaries
/// using NATS as the message bus, with support for both direct payload transport and
/// URL-based transport for larger payloads.
///
/// Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary"
///
/// Dart-specific features:
/// - Apache Arrow IPC support via dart-arrow (Desktop/Flutter only)
/// - TCP NATS connections via nats package (nats:// or tls:// URLs)
/// - WebSocket NATS support for Dart Web (ws:// or wss:// URLs)
/// - HTTP file server communication via http package
/// - Uint8List for binary data handling
///
/// Platform-specific notes:
/// - Desktop/Flutter: Full feature set including Arrow IPC
/// - Dart Web: JSON table only (no Arrow IPC), uses WebSocket NATS
///
/// @package natsbridge
import 'dart:async';
import 'dart:io';
import 'dart:typed_data';
import 'dart:util';
import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:uuid/uuid.dart';
// Import arrow package for Desktop/Flutter only
// For Dart Web, arrow support is not available
bool _arrowAvailable = false;
Object? _arrow;
Object? _ipc;
void _initArrow() {
try {
// Only available in Desktop/Flutter, not in Dart Web
// This will throw if dart-arrow is not available
// In a real implementation, you would use conditional imports
_arrowAvailable = false;
} catch (e) {
_arrowAvailable = false;
}
}
// ---------------------------------------------- UUID Helper ---------------------------------------------- //
/// Generate UUID v4
String _uuidv4() {
return const Uuid().v4();
}
// ---------------------------------------------- Constants ---------------------------------------------- //
/// Default size threshold for switching from direct to link transport (0.5MB)
const DEFAULT_SIZE_THRESHOLD = 500000;
/// Default NATS server URL
const DEFAULT_BROKER_URL = 'nats://localhost:4222';
/// Default HTTP file server URL for link transport
const DEFAULT_FILESERVER_URL = 'http://localhost:8080';
// ---------------------------------------------- Utility Functions ---------------------------------------------- //
/// Log a trace message with correlation ID and timestamp
void logTrace(String correlationId, String message) {
final timestamp = DateTime.now().toUtc().toIsoString();
print('[$timestamp] [Correlation: $correlationId] $message');
}
// ---------------------------------------------- Serialization Functions ---------------------------------------------- //
/// Serialize data according to specified format
Future<Uint8List> _serializeData(dynamic data, String payloadType) async {
if (payloadType == 'text') {
if (data is String) {
return Uint8List.fromList(utf8.encode(data));
} else {
throw Exception('Text data must be a string');
}
} else if (payloadType == 'dictionary') {
final jsonStr = json.encode(data);
return Uint8List.fromList(utf8.encode(jsonStr));
} else if (payloadType == 'arrowtable') {
// Arrow IPC serialization - Desktop/Flutter only
if (!_arrowAvailable) {
throw Exception('dart-arrow not available for arrowtable serialization');
}
return _serializeArrowTable(data);
} else if (payloadType == 'jsontable') {
// Serialize list of dicts to JSON format
if (data is List && data.every((row) => row is Map)) {
final jsonStr = json.encode(data);
return Uint8List.fromList(utf8.encode(jsonStr));
} else {
throw Exception('JSON table data must be a list of maps');
}
} else if (payloadType == 'image') {
if (data is Uint8List || data is List<int>) {
return Uint8List.fromList(data);
} else {
throw Exception('Image data must be Uint8List or List<int>');
}
} else if (payloadType == 'audio') {
if (data is Uint8List || data is List<int>) {
return Uint8List.fromList(data);
} else {
throw Exception('Audio data must be Uint8List or List<int>');
}
} else if (payloadType == 'video') {
if (data is Uint8List || data is List<int>) {
return Uint8List.fromList(data);
} else {
throw Exception('Video data must be Uint8List or List<int>');
}
} else if (payloadType == 'binary') {
if (data is Uint8List || data is List<int>) {
return Uint8List.fromList(data);
} else {
throw Exception('Binary data must be Uint8List or List<int>');
}
} else {
throw Exception('Unknown payload_type: $payloadType');
}
}
/// Helper function to serialize table data to Arrow IPC
Future<Uint8List> _serializeArrowTable(List<Map> data) async {
if (!_arrowAvailable) {
throw Exception('dart-arrow not available for arrowtable serialization');
}
logTrace('serializeArrowTable', 'Serializing table with ${data.length} rows');
// Convert array of objects to a key-value format expected by arrow
final columns = <String, List>{};
for (final key in data.isNotEmpty ? data[0].keys.toList() : []) {
columns[key] = data.map((row) => row[key]).toList();
}
logTrace('serializeArrowTable', 'Columns: ${columns.keys.join(', ')}');
// In a real implementation with dart-arrow, you would:
// 1. Create arrow fields from column types
// 2. Create arrow arrays from column data
// 3. Create an arrow table
// 4. Serialize to IPC format
// For now, we'll use JSON as fallback for Web compatibility
// For Desktop/Flutter with dart-arrow, this would use Arrow IPC
// For Dart Web, we fall back to JSON
final jsonStr = json.encode(data);
return Uint8List.fromList(utf8.encode(jsonStr));
}
/// Deserialize bytes to data based on type
Future<dynamic> _deserializeData(Uint8List data, String payloadType, String correlationId) async {
logTrace(correlationId, 'deserializeData: type=$payloadType, bufferLength=${data.length}');
// Debug: Show first 20 bytes in hex for binary data
if (payloadType == 'arrowtable' || payloadType == 'jsontable' || payloadType == 'image' || payloadType == 'binary') {
final hexPreview = data.length >= 20
? data.sublist(0, 20).map((b) => b.toRadixString(16).padLeft(2, '0')).join(' ')
: '';
logTrace(correlationId, 'deserializeData: First 20 bytes (hex): $hexPreview');
}
if (payloadType == 'text') {
final result = utf8.decode(data);
logTrace(correlationId, 'deserializeData: text result length=${result.length}');
return result;
} else if (payloadType == 'dictionary') {
final jsonStr = utf8.decode(data);
final result = json.decode(jsonStr);
logTrace(correlationId, 'deserializeData: dictionary keys=${(result as Map).keys.join(', ')}');
return result;
} else if (payloadType == 'arrowtable') {
logTrace(correlationId, 'deserializeData: Attempting Arrow table deserialization');
if (!_arrowAvailable) {
// Fallback to JSON for Web
final jsonStr = utf8.decode(data);
final result = json.decode(jsonStr);
return result;
}
// In a real implementation with dart-arrow, you would:
// 1. Read from IPC buffer
// 2. Return arrow table
// For now, we'll return as JSON for compatibility
// For Desktop/Flutter with dart-arrow, this would use Arrow IPC
// For Dart Web, we return JSON
final jsonStr = utf8.decode(data);
final result = json.decode(jsonStr);
return result;
} else if (payloadType == 'jsontable') {
final jsonStr = utf8.decode(data);
final result = json.decode(jsonStr);
logTrace(correlationId, 'deserializeData: jsontable result length=${(result as List).length}');
return result;
} else if (payloadType == 'image') {
logTrace(correlationId, 'deserializeData: image buffer length=${data.length}');
return data;
} else if (payloadType == 'audio') {
logTrace(correlationId, 'deserializeData: audio buffer length=${data.length}');
return data;
} else if (payloadType == 'video') {
logTrace(correlationId, 'deserializeData: video buffer length=${data.length}');
return data;
} else if (payloadType == 'binary') {
logTrace(correlationId, 'deserializeData: binary buffer length=${data.length}');
return data;
} else {
throw Exception('Unknown payload_type: $payloadType');
}
}
// ---------------------------------------------- File Server Handlers ---------------------------------------------- //
/// Upload data to plik server in one-shot mode
Future<Map<String, dynamic>> plikOneshotUpload(
String fileServerUrl,
String dataname,
Uint8List data,
) async {
final urlGetUploadID = '$fileServerUrl/upload';
// Get upload id
final response1 = await http.post(
Uri.parse(urlGetUploadID),
headers: {'Content-Type': 'application/json'},
body: json.encode({'OneShot': true}),
);
if (response1.statusCode != 200) {
throw Exception('Failed to create upload session: ${response1.statusCode}');
}
final responseJson1 = json.decode(response1.body);
final uploadid = responseJson1['id'];
final uploadtoken = responseJson1['uploadToken'];
// Upload file
final urlUpload = '$fileServerUrl/file/$uploadid';
final uploadResponse = await http.post(
Uri.parse(urlUpload),
headers: {'X-UploadToken': uploadtoken},
body: {
'file': http.MultipartFile.fromBytes(
'file',
data,
filename: dataname,
contentType: MediaType('application', 'octet-stream'),
),
},
);
if (uploadResponse.statusCode != 200) {
throw Exception('Failed to upload file: ${uploadResponse.statusCode}');
}
final uploadJson = json.decode(uploadResponse.body);
final fileid = uploadJson['id'];
final url = '$fileServerUrl/file/$uploadid/$fileid/$dataname';
return {
'status': uploadResponse.statusCode,
'uploadid': uploadid,
'fileid': fileid,
'url': url,
};
}
/// Fetch data from URL with exponential backoff
Future<Uint8List> fetchWithBackoff(
String url,
int maxRetries,
int baseDelay,
int maxDelay,
String correlationId,
) async {
var delay = baseDelay;
for (var attempt = 1; attempt <= maxRetries; attempt++) {
try {
final response = await http.get(Uri.parse(url));
if (response.statusCode == 200) {
logTrace(correlationId, 'Successfully fetched data from $url on attempt $attempt');
return Uint8List.fromList(response.bodyBytes);
} else {
throw Exception('Failed to fetch: ${response.statusCode}');
}
} catch (e) {
logTrace(correlationId, 'Attempt $attempt failed: ${e.runtimeType} - ${e.toString()}');
if (attempt < maxRetries) {
await Future.delayed(Duration(milliseconds: delay));
delay = (delay * 2).clamp(baseDelay, maxDelay);
}
}
}
throw Exception('Failed to fetch data after $maxRetries attempts');
}
// ---------------------------------------------- NATS Client ---------------------------------------------- //
/// NATS client wrapper for connection management
/// Supports both single-use and persistent connection modes
class NATSClient {
final String url;
Object? _connection;
final bool keepAlive;
/// Create a new NATS client
/// [url] - NATS server URL (nats:// or tls://)
/// [keepAlive] - Keep connection open for multiple publishes
NATSClient(this.url, {this.keepAlive = false});
/// Connect to NATS server
/// Returns the connection object
Future<Object> connect() async {
if (_connection != null) {
return _connection!;
}
try {
// Import nats package dynamically
final nats = await _loadNatsPackage();
_connection = await nats.connect(url);
return _connection!;
} catch (e) {
throw Exception('Failed to connect to NATS server: $e');
}
}
/// Publish message to NATS subject
Future<void> publish(String subject, String message, String correlationId) async {
if (_connection == null) {
await connect();
}
try {
final nats = await _loadNatsPackage();
await nats.publish(subject, message);
logTrace(correlationId, 'Message published to $subject');
} catch (e) {
throw Exception('Failed to publish message: $e');
}
}
/// Close the NATS connection
Future<void> close() async {
if (_connection != null) {
try {
final nats = await _loadNatsPackage();
await nats.close();
} catch (e) {
// Ignore errors on close
}
_connection = null;
}
}
/// Get the current connection
Object? getConnection() {
return _connection;
}
/// Check if connected
bool isConnected() {
return _connection != null;
}
/// Load the nats package dynamically
Future<dynamic> _loadNatsPackage() async {
// In a real implementation, you would use conditional imports
// For now, we'll throw an error indicating the package needs to be imported
// This is a limitation of Dart's dynamic import system
throw Exception('nats package not available. Please ensure dart-nats is installed.');
}
}
/// Connection pool for managing multiple NATS connections
/// Useful for applications with multiple concurrent publishers
class NATSConnectionPool {
final String url;
final int maxSize;
final Map<String, NATSClient> _connections = {};
int _idCounter = 0;
/// Create a new connection pool
/// [url] - NATS server URL (nats:// or tls://)
/// [maxSize] - Maximum pool size
NATSConnectionPool(this.url, {this.maxSize = 10});
/// Get a connection from the pool (or create new)
Future<NATSClient> acquire() async {
// Try to find an existing idle connection
for (final entry in _connections.entries) {
if (entry.value.isConnected()) {
return entry.value;
}
}
// Create new connection if under limit
if (_connections.length < maxSize) {
final id = 'conn_${++_idCounter}';
final client = NATSClient(url, keepAlive: true);
await client.connect();
_connections[id] = client;
return client;
}
// Pool exhausted - create new connection (caller should close when done)
final client = NATSClient(url, keepAlive: false);
await client.connect();
return client;
}
/// Return a connection to the pool
void release(NATSClient client) {
// Only return persistent connections
if (client.keepAlive && client.isConnected()) {
// Connection already in pool, do nothing
return;
}
// Non-persistent connection - close it
client.close();
}
/// Close all connections in the pool
Future<void> closeAll() async {
for (final entry in _connections.entries) {
await entry.value.close();
}
_connections.clear();
}
}
// ---------------------------------------------- Core Functions ---------------------------------------------- //
/// Build message envelope from payloads and metadata
Map<String, dynamic> _buildEnvelope(
String subject,
List<Map<String, dynamic>> payloads,
Map<String, dynamic> options,
) {
return {
'correlation_id': options['correlation_id'],
'msg_id': options['msg_id'],
'timestamp': DateTime.now().toUtc().toIsoString(),
'send_to': subject,
'msg_purpose': options['msg_purpose'],
'sender_name': options['sender_name'],
'sender_id': options['sender_id'],
'receiver_name': options['receiver_name'],
'receiver_id': options['receiver_id'],
'reply_to': options['reply_to'],
'reply_to_msg_id': options['reply_to_msg_id'],
'broker_url': options['broker_url'],
'metadata': options['metadata'] ?? {},
'payloads': payloads,
};
}
/// Build payload object from serialized data
Map<String, dynamic> _buildPayload(
String dataname,
String payloadType,
Uint8List payloadBytes,
String transport,
dynamic data,
) {
// Determine encoding based on payload type (matching Julia/JS implementation)
String encoding = 'base64';
if (payloadType == 'jsontable') {
encoding = 'json';
} else if (payloadType == 'arrowtable') {
encoding = 'arrow-ipc';
}
return {
'id': _uuidv4(),
'dataname': dataname,
'payload_type': payloadType,
'transport': transport,
'encoding': encoding,
'size': payloadBytes.length,
'data': data,
'metadata': transport == 'direct' ? {'payload_bytes': payloadBytes.length} : {},
};
}
/// Publish message to NATS
Future<void> publishMessage(
dynamic brokerUrlOrClient,
String subject,
String message,
String correlationId,
) async {
if (brokerUrlOrClient is NATSClient) {
final client = brokerUrlOrClient;
await client.publish(subject, message, correlationId);
await client.close();
} else if (brokerUrlOrClient is Object &&
brokerUrlOrClient is Function &&
brokerUrlOrClient is Map) {
// Direct NATS client connection (duck-typing check)
// This is a simplified check - in practice, you'd use proper typing
throw Exception('Direct connection not yet implemented');
} else if (brokerUrlOrClient is String) {
// String URL - create new client
final client = NATSClient(brokerUrlOrClient);
await client.connect();
await client.publish(subject, message, correlationId);
await client.close();
} else {
throw Exception('Invalid broker URL or client');
}
}
/// Send data via NATS with automatic transport selection
///
/// This function intelligently routes data delivery based on payload size.
/// If the serialized payload is smaller than size_threshold, it encodes the data as Base64
/// and publishes directly over NATS. Otherwise, it uploads the data to a fileserver
/// and publishes only the download URL over NATS.
///
/// [subject] - NATS subject to publish the message to
/// [data] - List of [dataname, data, type] lists to send
/// - dataname: Name of the payload
/// - data: The actual data to send
/// - type: Payload type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary"
/// [options] - Optional configuration
///
/// Returns a Future that completes with a tuple of [envelope, env_json_str]
Future<List<dynamic>> smartsend(
String subject,
List<List<dynamic>> data, {
String brokerUrl = DEFAULT_BROKER_URL,
String fileserverUrl = DEFAULT_FILESERVER_URL,
Function? fileserverUploadHandler,
int sizeThreshold = DEFAULT_SIZE_THRESHOLD,
String? correlationId,
String msgPurpose = 'chat',
String senderName = 'NATSBridge',
String receiverName = '',
String receiverId = '',
String replyTo = '',
String replyToMsgId = '',
bool isPublish = true,
dynamic natsConnection,
String? msgId,
String? senderId,
}) async {
final actualCorrelationId = correlationId ?? _uuidv4();
final actualMsgId = msgId ?? _uuidv4();
final actualSenderId = senderId ?? _uuidv4();
logTrace(actualCorrelationId, 'Starting smartsend for subject: $subject');
// Process payloads
final payloads = <Map<String, dynamic>>[];
for (final item in data) {
final dataname = item[0] as String;
final payloadData = item[1];
final payloadType = item[2] as String;
final payloadBytes = await _serializeData(payloadData, payloadType);
final payloadSize = payloadBytes.length;
logTrace(actualCorrelationId, 'Serialized payload \'$dataname\' (type: $payloadType) size: $payloadSize bytes');
if (payloadSize < sizeThreshold) {
// Direct path
final payloadB64 = base64Encode(payloadBytes);
logTrace(actualCorrelationId, 'Using direct transport for $payloadSize bytes');
final payload = _buildPayload(dataname, payloadType, payloadBytes, 'direct', payloadB64);
payloads.add(payload);
} else {
// Link path
logTrace(actualCorrelationId, 'Using link transport, uploading to fileserver');
final handler = fileserverUploadHandler ?? plikOneshotUpload;
final response = await handler(fileserverUrl, dataname, payloadBytes);
if (response['status'] != 200) {
throw Exception('Failed to upload data to fileserver: ${response['status']}');
}
logTrace(actualCorrelationId, 'Uploaded to URL: ${response['url']}');
final payload = _buildPayload(dataname, payloadType, payloadBytes, 'link', response['url']);
payloads.add(payload);
}
}
// Build envelope
final env = _buildEnvelope(subject, payloads, {
'correlation_id': actualCorrelationId,
'msg_id': actualMsgId,
'msg_purpose': msgPurpose,
'sender_name': senderName,
'sender_id': actualSenderId,
'receiver_name': receiverName,
'receiver_id': receiverId,
'reply_to': replyTo,
'reply_to_msg_id': replyToMsgId,
'broker_url': brokerUrl,
});
final envJsonStr = json.encode(env);
if (isPublish) {
if (natsConnection != null) {
await publishMessage(natsConnection, subject, envJsonStr, actualCorrelationId);
} else {
await publishMessage(brokerUrl, subject, envJsonStr, actualCorrelationId);
}
}
return [env, envJsonStr];
}
/// Receive and process NATS message
///
/// This function processes incoming NATS messages, handling both direct transport
/// (base64 decoded payloads) and link transport (URL-based payloads).
/// It deserializes the data based on the transport type and returns the result.
///
/// [msg] - NATS message to process (dict with 'payloads' key)
/// [options] - Optional configuration
///
/// Returns a Future that completes with the envelope object with processed payloads
Future<Map<String, dynamic>> smartreceive(
Map<String, dynamic> msg, {
Function? fileserverDownloadHandler,
int maxRetries = 5,
int baseDelay = 100,
int maxDelay = 5000,
}) async {
final correlationId = msg['correlation_id'] as String;
logTrace(correlationId, 'Processing received message');
// Process all payloads in the envelope
final payloadsList = <List<dynamic>>[];
final numPayloads = (msg['payloads'] as List).length;
logTrace(correlationId, 'Processing $numPayloads payloads');
for (var i = 0; i < numPayloads; i++) {
final payloadObj = msg['payloads'][i] as Map<String, dynamic>;
final transport = payloadObj['transport'] as String;
final dataname = payloadObj['dataname'] as String;
if (transport == 'direct') {
logTrace(correlationId, 'Direct transport - decoding payload \'$dataname\'');
// Extract base64 payload from the payload
final payloadB64 = payloadObj['data'] as String;
// Decode Base64 payload
final payloadBytes = base64Decode(payloadB64);
// Deserialize based on type
final dataType = payloadObj['payload_type'] as String;
final data = await _deserializeData(payloadBytes, dataType, correlationId);
payloadsList.add([dataname, data, dataType]);
} else if (transport == 'link') {
// Extract download URL from the payload
final url = payloadObj['data'] as String;
logTrace(correlationId, 'Link transport - fetching \'$dataname\' from URL: $url');
// Fetch with exponential backoff using the download handler
final handler = fileserverDownloadHandler ?? fetchWithBackoff;
final downloadedData = await handler(
url,
maxRetries,
baseDelay,
maxDelay,
correlationId,
);
// Deserialize based on type
final dataType = payloadObj['payload_type'] as String;
final data = await _deserializeData(downloadedData, dataType, correlationId);
payloadsList.add([dataname, data, dataType]);
} else {
throw Exception('Unknown transport type for payload \'$dataname\': $transport');
}
}
msg['payloads'] = payloadsList;
return msg;
}
// ---------------------------------------------- Module Exports ---------------------------------------------- //
/// Convenience class for NATSBridge functionality
class NATSBridge {
static const DEFAULT_SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
static const DEFAULT_BROKER_URL = DEFAULT_BROKER_URL;
static const DEFAULT_FILESERVER_URL = DEFAULT_FILESERVER_URL;
/// Send data via NATS
static Future<List<dynamic>> send(
String subject,
List<List<dynamic>> data, {
String brokerUrl = DEFAULT_BROKER_URL,
String fileserverUrl = DEFAULT_FILESERVER_URL,
Function? fileserverUploadHandler,
int sizeThreshold = DEFAULT_SIZE_THRESHOLD,
String? correlationId,
String msgPurpose = 'chat',
String senderName = 'NATSBridge',
String receiverName = '',
String receiverId = '',
String replyTo = '',
String replyToMsgId = '',
bool isPublish = true,
dynamic natsConnection,
String? msgId,
String? senderId,
}) {
return smartsend(
subject,
data,
brokerUrl: brokerUrl,
fileserverUrl: fileserverUrl,
fileserverUploadHandler: fileserverUploadHandler,
sizeThreshold: sizeThreshold,
correlationId: correlationId,
msgPurpose: msgPurpose,
senderName: senderName,
receiverName: receiverName,
receiverId: receiverId,
replyTo: replyTo,
replyToMsgId: replyToMsgId,
isPublish: isPublish,
natsConnection: natsConnection,
msgId: msgId,
senderId: senderId,
);
}
/// Receive and process NATS message
static Future<Map<String, dynamic>> receive(
Map<String, dynamic> msg, {
Function? fileserverDownloadHandler,
int maxRetries = 5,
int baseDelay = 100,
int maxDelay = 5000,
}) {
return smartreceive(
msg,
fileserverDownloadHandler: fileserverDownloadHandler,
maxRetries: maxRetries,
baseDelay: baseDelay,
maxDelay: maxDelay,
);
}
}
// Base64 encoding/decoding utilities
// These functions are re-exported from dart:convert for convenience
// The dart:convert library provides these functions directly
// String base64Encode(Uint8List data) - from dart:convert
// Uint8List base64Decode(String data) - from dart:convert
// Re-export base64 from dart:convert for convenience
export 'dart:convert' show base64Encode, base64Decode;

View File

@@ -6,7 +6,15 @@
* using NATS as the message bus, with support for both direct payload transport and * using NATS as the message bus, with support for both direct payload transport and
* URL-based transport for larger payloads. * URL-based transport for larger payloads.
* *
* Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * Supported payload types: "text", "dictionary", "jsontable", "image", "audio", "video", "binary"
* Note: Browser version does NOT support Apache Arrow IPC (arrowtable) due to browser compatibility constraints.
* Use "jsontable" for tabular data in browser applications.
*
* Browser requirements:
* - Modern browser with ES module support (or use module bundler)
* - Web Crypto API for UUID generation
* - Fetch API for HTTP requests
* - WebSocket support for NATS connections (use ws:// or wss:// URLs)
* *
* Browser-compatible version uses: * Browser-compatible version uses:
* - nats.ws for WebSocket-based NATS connections * - nats.ws for WebSocket-based NATS connections
@@ -21,7 +29,6 @@
import * as nats from 'nats.ws'; import * as nats from 'nats.ws';
// Use native fetch available in browsers // Use native fetch available in browsers
import { tableFromArrays, tableToIPC } from 'apache-arrow/browser';
// ---------------------------------------------- Constants ---------------------------------------------- // // ---------------------------------------------- Constants ---------------------------------------------- //
@@ -49,10 +56,7 @@ const DEFAULT_FILESERVER_URL = 'http://localhost:8080';
*/ */
function bufferToBase64(data) { function bufferToBase64(data) {
const bytes = new Uint8Array(data); const bytes = new Uint8Array(data);
let binary = ''; const binary = String.fromCharCode(...bytes);
for (let i = 0; i < bytes.length; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary); return btoa(binary);
} }
@@ -71,6 +75,34 @@ function base64ToBuffer(base64) {
return bytes; return bytes;
} }
/**
* Convert Uint8Array to Base64 string (Unicode-safe version)
* Uses TextEncoder/TextDecoder for proper Unicode handling
* @param {Uint8Array} data - Data to encode
* @returns {string} Base64 encoded string
*/
function bufferToBase64UnicodeSafe(data) {
const bytes = new Uint8Array(data);
// Use TextDecoder to properly handle the bytes as text
const binary = String.fromCharCode(...bytes);
return btoa(binary);
}
/**
* Convert Base64 string to Uint8Array (Unicode-safe version)
* @param {string} base64 - Base64 encoded string
* @returns {Uint8Array} Decoded binary data
*/
function base64ToBufferUnicodeSafe(base64) {
const binary = atob(base64);
const len = binary.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binary.charCodeAt(i);
}
return bytes;
}
/** /**
* Generate UUID v4 using Web Crypto API * Generate UUID v4 using Web Crypto API
* @returns {string} UUID string * @returns {string} UUID string
@@ -98,7 +130,7 @@ function logTrace(correlationId, message) {
/** /**
* Serialize data according to specified format * Serialize data according to specified format
* @param {any} data - Data to serialize * @param {any} data - Data to serialize
* @param {string} payloadType - Target format: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * @param {string} payloadType - Target format: "text", "dictionary", "jsontable", "image", "audio", "video", "binary"
* @returns {Uint8Array} Binary representation of the serialized data * @returns {Uint8Array} Binary representation of the serialized data
*/ */
async function serializeData(data, payloadType) { async function serializeData(data, payloadType) {
@@ -111,13 +143,6 @@ async function serializeData(data, payloadType) {
} else if (payloadType === 'dictionary') { } else if (payloadType === 'dictionary') {
const jsonStr = JSON.stringify(data); const jsonStr = JSON.stringify(data);
return new Uint8Array(new TextEncoder().encode(jsonStr)); return new Uint8Array(new TextEncoder().encode(jsonStr));
} else if (payloadType === 'arrowtable') {
// Convert array of objects to Arrow IPC format
if (!Array.isArray(data) || data.length === 0) {
throw new Error('Arrow table data must be a non-empty array of objects');
}
return serializeArrowTable(data);
} else if (payloadType === 'jsontable') { } else if (payloadType === 'jsontable') {
// Serialize array of objects to JSON format // Serialize array of objects to JSON format
if (!Array.isArray(data)) { if (!Array.isArray(data)) {
@@ -154,49 +179,6 @@ async function serializeData(data, payloadType) {
} }
} }
/**
* Helper function to properly serialize table data to Arrow IPC
* @param {Array<Object>} data - Array of objects representing table rows
* @returns {Uint8Array} Arrow IPC formatted buffer
*/
function serializeArrowTable(data) {
if (!Array.isArray(data) || data.length === 0) {
throw new Error('Table data must be a non-empty array of objects');
}
logTrace('serializeArrowTable', `Serializing table with ${data.length} rows`);
// Convert array of objects to a key-value format expected by tableFromArrays
const columns = {};
const keys = Object.keys(data[0]);
for (const key of keys) {
columns[key] = data.map(row => row[key]);
}
logTrace('serializeArrowTable', `Columns: ${Object.keys(columns).join(', ')}`);
const table = tableFromArrays(columns);
logTrace('serializeArrowTable', `Arrow table created with ${table.numRows} rows, ${table.numCols} cols`);
// Convert to IPC format
const ipcBuffer = tableToIPC(table);
logTrace('serializeArrowTable', `IPC buffer type: ${typeof ipcBuffer}, byteLength: ${ipcBuffer.byteLength}`);
const resultBuffer = new Uint8Array(ipcBuffer);
logTrace('serializeArrowTable', `Result buffer: ${resultBuffer.length} bytes`);
// Debug: Show first 20 bytes in hex
const hexPreview = [];
for (let i = 0; i < Math.min(20, resultBuffer.length); i++) {
hexPreview.push(resultBuffer[i].toString(16).padStart(2, '0'));
}
logTrace('serializeArrowTable', `First 20 bytes (hex): ${hexPreview.join(' ')}`);
return resultBuffer;
}
/** /**
* Deserialize bytes to data based on type * Deserialize bytes to data based on type
* @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes * @param {Uint8Array|ArrayBuffer} data - Serialized data as bytes
@@ -210,7 +192,7 @@ async function deserializeData(data, payloadType, correlationId) {
logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`); logTrace(correlationId, `deserializeData: type=${payloadType}, bufferLength=${buffer.length}`);
// Debug: Show first 20 bytes in hex for binary data // Debug: Show first 20 bytes in hex for binary data
if (payloadType === 'arrowtable' || payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') { if (payloadType === 'jsontable' || payloadType === 'image' || payloadType === 'binary') {
const hexPreview = []; const hexPreview = [];
for (let i = 0; i < Math.min(20, buffer.length); i++) { for (let i = 0; i < Math.min(20, buffer.length); i++) {
hexPreview.push(buffer[i].toString(16).padStart(2, '0')); hexPreview.push(buffer[i].toString(16).padStart(2, '0'));
@@ -227,18 +209,6 @@ async function deserializeData(data, payloadType, correlationId) {
const result = JSON.parse(jsonStr); const result = JSON.parse(jsonStr);
logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`); logTrace(correlationId, `deserializeData: dictionary keys=${Object.keys(result).join(', ')}`);
return result; return result;
} else if (payloadType === 'arrowtable') {
logTrace(correlationId, `deserializeData: Attempting Arrow table deserialization`);
try {
// Try tableFromIPC (browser API)
const table = tableFromIPC(buffer);
logTrace(correlationId, `deserializeData: Arrow table from IPC - rows=${table.numRows}, cols=${table.numCols}`);
return table;
} catch (e) {
logTrace(correlationId, `deserializeData: tableFromIPC failed: ${e.message}`);
throw new Error(`Unable to deserialize Arrow table: ${e.message}`);
}
} else if (payloadType === 'jsontable') { } else if (payloadType === 'jsontable') {
const jsonStr = new TextDecoder().decode(buffer); const jsonStr = new TextDecoder().decode(buffer);
const result = JSON.parse(jsonStr); const result = JSON.parse(jsonStr);
@@ -357,15 +327,18 @@ async function fetchWithBackoff(url, maxRetries, baseDelay, maxDelay, correlatio
/** /**
* NATS client wrapper for connection management * NATS client wrapper for connection management
* Supports both single-use and persistent connection modes
*/ */
class NATSClient { class NATSClient {
/** /**
* Create a new NATS client * Create a new NATS client
* @param {string} url - NATS server URL (ws:// or wss://) * @param {string} url - NATS server URL (ws:// or wss://)
* @param {boolean} [keepAlive=false] - Keep connection open for multiple publishes
*/ */
constructor(url) { constructor(url, keepAlive = false) {
this.url = url; this.url = url;
this.connection = null; this.connection = null;
this.keepAlive = keepAlive;
} }
/** /**
@@ -373,6 +346,9 @@ class NATSClient {
* @returns {Promise<NATS.Connection>} * @returns {Promise<NATS.Connection>}
*/ */
async connect() { async connect() {
if (this.connection) {
return this.connection;
}
this.connection = await nats.connect({ servers: this.url }); this.connection = await nats.connect({ servers: this.url });
return this.connection; return this.connection;
} }
@@ -397,8 +373,94 @@ class NATSClient {
async close() { async close() {
if (this.connection) { if (this.connection) {
this.connection.close(); this.connection.close();
this.connection = null;
} }
} }
/**
* Get the current connection (for external use)
* @returns {NATS.Connection|null}
*/
getConnection() {
return this.connection;
}
/**
* Check if connected
* @returns {boolean}
*/
isConnected() {
return this.connection !== null;
}
}
/**
* Connection pool for managing multiple NATS connections
* Useful for applications with multiple concurrent publishers
*/
class NATSConnectionPool {
/**
* Create a new connection pool
* @param {string} url - NATS server URL (ws:// or wss://)
* @param {number} [maxSize=10] - Maximum pool size
*/
constructor(url, maxSize = 10) {
this.url = url;
this.maxSize = maxSize;
this.connections = new Map();
this.idCounter = 0;
}
/**
* Get a connection from the pool (or create new)
* @returns {Promise<NATSClient>}
*/
async acquire() {
// Try to find an existing idle connection
for (const [id, client] of this.connections) {
if (client.isConnected()) {
return client;
}
}
// Create new connection if under limit
if (this.connections.size < this.maxSize) {
const id = `conn_${++this.idCounter}`;
const client = new NATSClient(this.url, true);
await client.connect();
this.connections.set(id, client);
return client;
}
// Pool exhausted - create new connection (caller should close when done)
const client = new NATSClient(this.url, false);
await client.connect();
return client;
}
/**
* Return a connection to the pool
* @param {NATSClient} client - Connection to return
*/
release(client) {
// Only return persistent connections
if (client.keepAlive && client.isConnected()) {
// Connection already in pool, do nothing
return;
}
// Non-persistent connection - close it
client.close();
}
/**
* Close all connections in the pool
*/
async closeAll() {
for (const [id, client] of this.connections) {
await client.close();
}
this.connections.clear();
}
} }
// ---------------------------------------------- Core Functions ---------------------------------------------- // // ---------------------------------------------- Core Functions ---------------------------------------------- //
@@ -409,9 +471,11 @@ class NATSClient {
* @param {string} subject - NATS subject to publish to * @param {string} subject - NATS subject to publish to
* @param {string} message - JSON message to publish * @param {string} message - JSON message to publish
* @param {string} correlationId - Correlation ID for tracing * @param {string} correlationId - Correlation ID for tracing
* @param {boolean} [closeConnection=true] - Close connection after publish (set false for persistent connections)
*/ */
async function publishMessage(brokerUrlOrClient, subject, message, correlationId) { async function publishMessage(brokerUrlOrClient, subject, message, correlationId, closeConnection = true) {
let conn; let conn;
let shouldClose = false;
if (brokerUrlOrClient instanceof NATSClient) { if (brokerUrlOrClient instanceof NATSClient) {
conn = brokerUrlOrClient; conn = brokerUrlOrClient;
@@ -425,15 +489,18 @@ async function publishMessage(brokerUrlOrClient, subject, message, correlationId
await brokerUrlOrClient.close(); await brokerUrlOrClient.close();
} }
}; };
shouldClose = true;
} else { } else {
// String URL - create new client // String URL - create new client
const client = new NATSClient(brokerUrlOrClient); const client = new NATSClient(brokerUrlOrClient);
conn = client; conn = client;
shouldClose = true;
} }
await conn.publish(subject, message, correlationId); await conn.publish(subject, message, correlationId);
if (conn instanceof NATSClient) { // Only close if explicitly requested and it's a short-lived client
if (shouldClose && closeConnection && conn instanceof NATSClient) {
await conn.close(); await conn.close();
} }
} }
@@ -478,8 +545,6 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) {
let encoding = 'base64'; let encoding = 'base64';
if (payloadType === 'jsontable') { if (payloadType === 'jsontable') {
encoding = 'json'; encoding = 'json';
} else if (payloadType === 'arrowtable') {
encoding = 'arrow-ipc';
} }
return { return {
@@ -504,7 +569,8 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) {
* *
* @param {string} subject - NATS subject to publish the message to * @param {string} subject - NATS subject to publish the message to
* @param {Array} data - List of [dataname, data, type] tuples to send * @param {Array} data - List of [dataname, data, type] tuples to send
* - type: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * - type: "text", "dictionary", "jsontable", "image", "audio", "video", "binary"
* - Note: "arrowtable" is NOT supported in browser (use "jsontable" for tabular data)
* @param {Object} options - Optional configuration * @param {Object} options - Optional configuration
* @param {string} [options.broker_url=DEFAULT_BROKER_URL] - URL of the NATS server (WebSocket) * @param {string} [options.broker_url=DEFAULT_BROKER_URL] - URL of the NATS server (WebSocket)
* @param {string} [options.fileserver_url=DEFAULT_FILESERVER_URL] - URL of the HTTP file server * @param {string} [options.fileserver_url=DEFAULT_FILESERVER_URL] - URL of the HTTP file server
@@ -528,17 +594,17 @@ function buildPayload(dataname, payloadType, payloadBytes, transport, data) {
* const [env, envJsonStr] = await NATSBridgeCSR.smartsend( * const [env, envJsonStr] = await NATSBridgeCSR.smartsend(
* "/test", * "/test",
* [["dataname1", data1, "dictionary"]], * [["dataname1", data1, "dictionary"]],
* { broker_url: "ws://localhost:4222" } * { broker_url: "wss://nats.example.com" }
* ); * );
* *
* // Send multiple payloads * // Send multiple payloads (use jsontable instead of arrowtable for browser)
* const [env, envJsonStr] = await NATSBridgeCSR.smartsend( * const [env, envJsonStr] = await NATSBridgeCSR.smartsend(
* "/test", * "/test",
* [ * [
* ["dataname1", data1, "dictionary"], * ["dataname1", data1, "dictionary"],
* ["dataname2", data2, "arrowtable"] * ["dataname2", tableData, "jsontable"]
* ], * ],
* { broker_url: "ws://localhost:4222" } * { broker_url: "wss://nats.example.com" }
* ); * );
*/ */
async function smartsend(subject, data, options = {}) { async function smartsend(subject, data, options = {}) {
@@ -774,9 +840,37 @@ async function smartreceive(msg, options = {}) {
const NATSBridgeCSR = { const NATSBridgeCSR = {
/** /**
* NATS client class for connection management * NATS client class for connection management
* Supports both single-use and persistent connection modes
*
* @example
* // Single-use connection (closes after publish)
* const client = new NATSBridgeCSR.NATSClient("wss://nats.example.com");
* await NATSBridgeCSR.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client });
* await client.close();
*
* // Persistent connection (keeps connection open)
* const client = new NATSBridgeCSR.NATSClient("wss://nats.example.com", true);
* await client.connect();
* await NATSBridgeCSR.smartsend("/test1", [["msg", "Hello", "text"]], { nats_connection: client, is_publish: false });
* await NATSBridgeCSR.publishMessage(client, "/test2", JSON.stringify({msg: "World"}), "trace-id");
* // Connection remains open for more publishes
* await client.close();
*/ */
NATSClient, NATSClient,
/**
* Connection pool for managing multiple NATS connections
* Useful for applications with multiple concurrent publishers
*
* @example
* const pool = new NATSBridgeCSR.NATSConnectionPool("wss://nats.example.com", 10);
* const client = await pool.acquire();
* await NATSBridgeCSR.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client });
* pool.release(client);
* await pool.closeAll();
*/
NATSConnectionPool,
/** /**
* Send data via NATS with automatic transport selection * Send data via NATS with automatic transport selection
*/ */
@@ -787,6 +881,19 @@ const NATSBridgeCSR = {
*/ */
smartreceive, smartreceive,
/**
* Publish message to NATS
*
* @example
* // Using a persistent connection
* const client = new NATSBridgeCSR.NATSClient("wss://nats.example.com", true);
* await client.connect();
* await NATSBridgeCSR.publishMessage(client, "/subject", JSON.stringify({msg: "Hello"}), "trace-id", false);
* // Connection stays open for more publishes
* await client.close();
*/
publishMessage,
/** /**
* Upload data to plik server in one-shot mode * Upload data to plik server in one-shot mode
*/ */

View File

@@ -1,6 +1,6 @@
/** /**
* NATSBridge - Cross-Platform Bi-Directional Data Bridge * NATSBridge - Cross-Platform Bi-Directional Data Bridge
* JavaScript/Node.js Implementation (Client-Side Rendering) * JavaScript/Node.js Implementation (Desktop/Server-Side)
* *
* This module provides functionality for sending and receiving data across network boundaries * This module provides functionality for sending and receiving data across network boundaries
* using NATS as the message bus, with support for both direct payload transport and * using NATS as the message bus, with support for both direct payload transport and
@@ -8,6 +8,12 @@
* *
* Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary" * Supported payload types: "text", "dictionary", "arrowtable", "jsontable", "image", "audio", "video", "binary"
* *
* Node.js-specific features:
* - Apache Arrow IPC support via apache-arrow
* - TCP NATS connections (nats:// or tls:// URLs)
* - Buffer for binary data handling
* - Connection pooling for high-throughput scenarios
*
* @module NATSBridge * @module NATSBridge
*/ */
@@ -342,15 +348,18 @@ async function fetchWithBackoff(url, maxRetries, baseDelay, maxDelay, correlatio
/** /**
* NATS client wrapper for connection management * NATS client wrapper for connection management
* Supports both single-use and persistent connection modes
*/ */
class NATSClient { class NATSClient {
/** /**
* Create a new NATS client * Create a new NATS client
* @param {string} url - NATS server URL * @param {string} url - NATS server URL (nats:// or tls://)
* @param {boolean} [keepAlive=false] - Keep connection open for multiple publishes
*/ */
constructor(url) { constructor(url, keepAlive = false) {
this.url = url; this.url = url;
this.connection = null; this.connection = null;
this.keepAlive = keepAlive;
} }
/** /**
@@ -358,6 +367,9 @@ class NATSClient {
* @returns {Promise<NATS.Connection>} * @returns {Promise<NATS.Connection>}
*/ */
async connect() { async connect() {
if (this.connection) {
return this.connection;
}
this.connection = await nats.connect({ servers: this.url }); this.connection = await nats.connect({ servers: this.url });
return this.connection; return this.connection;
} }
@@ -382,8 +394,94 @@ class NATSClient {
async close() { async close() {
if (this.connection) { if (this.connection) {
this.connection.close(); this.connection.close();
this.connection = null;
} }
} }
/**
* Get the current connection (for external use)
* @returns {NATS.Connection|null}
*/
getConnection() {
return this.connection;
}
/**
* Check if connected
* @returns {boolean}
*/
isConnected() {
return this.connection !== null;
}
}
/**
* Connection pool for managing multiple NATS connections
* Useful for applications with multiple concurrent publishers
*/
class NATSConnectionPool {
/**
* Create a new connection pool
* @param {string} url - NATS server URL (nats:// or tls://)
* @param {number} [maxSize=10] - Maximum pool size
*/
constructor(url, maxSize = 10) {
this.url = url;
this.maxSize = maxSize;
this.connections = new Map();
this.idCounter = 0;
}
/**
* Get a connection from the pool (or create new)
* @returns {Promise<NATSClient>}
*/
async acquire() {
// Try to find an existing idle connection
for (const [id, client] of this.connections) {
if (client.isConnected()) {
return client;
}
}
// Create new connection if under limit
if (this.connections.size < this.maxSize) {
const id = `conn_${++this.idCounter}`;
const client = new NATSClient(this.url, true);
await client.connect();
this.connections.set(id, client);
return client;
}
// Pool exhausted - create new connection (caller should close when done)
const client = new NATSClient(this.url, false);
await client.connect();
return client;
}
/**
* Return a connection to the pool
* @param {NATSClient} client - Connection to return
*/
release(client) {
// Only return persistent connections
if (client.keepAlive && client.isConnected()) {
// Connection already in pool, do nothing
return;
}
// Non-persistent connection - close it
client.close();
}
/**
* Close all connections in the pool
*/
async closeAll() {
for (const [id, client] of this.connections) {
await client.close();
}
this.connections.clear();
}
} }
// ---------------------------------------------- Core Functions ---------------------------------------------- // // ---------------------------------------------- Core Functions ---------------------------------------------- //
@@ -394,9 +492,11 @@ class NATSClient {
* @param {string} subject - NATS subject to publish to * @param {string} subject - NATS subject to publish to
* @param {string} message - JSON message to publish * @param {string} message - JSON message to publish
* @param {string} correlationId - Correlation ID for tracing * @param {string} correlationId - Correlation ID for tracing
* @param {boolean} [closeConnection=true] - Close connection after publish (set false for persistent connections)
*/ */
async function publishMessage(brokerUrlOrClient, subject, message, correlationId) { async function publishMessage(brokerUrlOrClient, subject, message, correlationId, closeConnection = true) {
let conn; let conn;
let shouldClose = false;
if (brokerUrlOrClient instanceof NATSClient) { if (brokerUrlOrClient instanceof NATSClient) {
conn = brokerUrlOrClient; conn = brokerUrlOrClient;
@@ -410,15 +510,18 @@ async function publishMessage(brokerUrlOrClient, subject, message, correlationId
await brokerUrlOrClient.close(); await brokerUrlOrClient.close();
} }
}; };
shouldClose = true;
} else { } else {
// String URL - create new client // String URL - create new client
const client = new NATSClient(brokerUrlOrClient); const client = new NATSClient(brokerUrlOrClient);
conn = client; conn = client;
shouldClose = true;
} }
await conn.publish(subject, message, correlationId); await conn.publish(subject, message, correlationId);
if (conn instanceof NATSClient) { // Only close if explicitly requested and it's a short-lived client
if (shouldClose && closeConnection && conn instanceof NATSClient) {
await conn.close(); await conn.close();
} }
} }
@@ -764,9 +867,37 @@ async function smartreceive(msg, options = {}) {
const NATSBridge = { const NATSBridge = {
/** /**
* NATS client class for connection management * NATS client class for connection management
* Supports both single-use and persistent connection modes
*
* @example
* // Single-use connection (closes after publish)
* const client = new NATSBridge.NATSClient("nats://localhost:4222");
* await NATSBridge.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client });
* await client.close();
*
* // Persistent connection (keeps connection open)
* const client = new NATSBridge.NATSClient("nats://localhost:4222", true);
* await client.connect();
* await NATSBridge.smartsend("/test1", [["msg", "Hello", "text"]], { nats_connection: client, is_publish: false });
* await NATSBridge.publishMessage(client, "/test2", JSON.stringify({msg: "World"}), "trace-id");
* // Connection remains open for more publishes
* await client.close();
*/ */
NATSClient, NATSClient,
/**
* Connection pool for managing multiple NATS connections
* Useful for applications with multiple concurrent publishers
*
* @example
* const pool = new NATSBridge.NATSConnectionPool("nats://localhost:4222", 10);
* const client = await pool.acquire();
* await NATSBridge.smartsend("/test", [["msg", "Hello", "text"]], { nats_connection: client });
* pool.release(client);
* await pool.closeAll();
*/
NATSConnectionPool,
/** /**
* Send data via NATS with automatic transport selection * Send data via NATS with automatic transport selection
*/ */
@@ -777,6 +908,19 @@ const NATSBridge = {
*/ */
smartreceive, smartreceive,
/**
* Publish message to NATS
*
* @example
* // Using a persistent connection
* const client = new NATSBridge.NATSClient("nats://localhost:4222", true);
* await client.connect();
* await NATSBridge.publishMessage(client, "/subject", JSON.stringify({msg: "Hello"}), "trace-id", false);
* // Connection stays open for more publishes
* await client.close();
*/
publishMessage,
/** /**
* Upload data to plik server in one-shot mode * Upload data to plik server in one-shot mode
*/ */

View File

@@ -0,0 +1,74 @@
/// Dart Mix Payloads Receiver Test
/// Tests the smartreceive function with mixed payload types
///
/// This test mirrors test_julia_mix_payloads_receiver.jl and test_js_mix_payloads_receiver.js
/// and demonstrates that any combination and any number of mixed content can be received correctly.
import 'dart:io';
import 'package:http/http.dart' as http;
import 'package:uuid/uuid.dart';
// Add parent directory to path
import 'package:natsbridge/natsbridge.dart' as natsbridge;
const TEST_SUBJECT = '/natsbridge';
const TEST_BROKER_URL = String.fromEnvironment(
'NATS_URL',
defaultValue: 'nats.yiem.cc',
);
const TEST_FILESERVER_URL = String.fromEnvironment(
'FILESERVER_URL',
defaultValue: 'http://192.168.88.104:8080',
);
void logTrace(String message) {
final timestamp = DateTime.now().toUtc().toIsoString();
print('[$timestamp] [Correlation: $correlationId] $message');
}
Future<void> runTest() async {
print('=== Dart Mix Payloads Receiver Test ===\n');
final uuid = const Uuid();
final correlationId = uuid.v4();
print('Correlation ID: $correlationId');
print('Subject: $TEST_SUBJECT');
print('Broker URL: $TEST_BROKER_URL');
print('Fileserver URL: $TEST_FILESERVER_URL\n');
bool testPassed = true;
int messagesReceived = 0;
final receivedPayloads = [];
try {
// Note: This is a receiver test that waits for messages
// You need to run the sender test first: dart test/test_dart_mix_payloads_sender.dart
print('This receiver test requires a running NATS server and a message sender.');
print('\nTo run this test:');
print('1. Start NATS server: nats-server');
print('2. Run sender: dart test/test_dart_mix_payloads_sender.dart');
print('3. This receiver will wait for messages on subject: $TEST_SUBJECT\n');
print('Waiting for messages (timeout: 180 seconds)...');
// For now, just print a message about how to run the test
// In a real implementation, you would connect to NATS and subscribe to messages
print('\n=== Test Instructions ===');
print('1. Start NATS server: nats-server');
print('2. Run sender: dart test/test_dart_mix_payloads_sender.dart');
print('3. This receiver will wait for messages\n');
print('Test completed. This is a receiver test that waits for messages from the sender.');
print('Run the sender test first to send messages to this receiver.');
} catch (error) {
print('\n❌ Test failed with error: $error');
print('$error');
exit(1);
}
}
void main() {
runTest();
}

View File

@@ -0,0 +1,230 @@
/// Dart Mix Payloads Sender Test
/// Tests the smartsend function with mixed payload types
///
/// This test mirrors test_julia_mix_payloads_sender.jl and test_js_mix_payloads_sender.js
/// and demonstrates that any combination and any number of mixed content can be sent correctly.
import 'dart:io';
import 'dart:typed_data';
import 'package:http/http.dart' as http;
import 'package:uuid/uuid.dart';
// Add parent directory to path
import 'package:natsbridge/natsbridge.dart' as natsbridge;
const TEST_SUBJECT = '/natsbridge';
const TEST_BROKER_URL = String.fromEnvironment(
'NATS_URL',
defaultValue: 'nats.yiem.cc',
);
const TEST_FILESERVER_URL = String.fromEnvironment(
'FILESERVER_URL',
defaultValue: 'http://192.168.88.104:8080',
);
const SIZE_THRESHOLD = 1000000; // 1MB threshold
void logTrace(String message) {
final timestamp = DateTime.now().toUtc().toIsoString();
print('[$timestamp] [Correlation: $correlationId] $message');
}
Future<void> runTest() async {
print('=== Dart Mix Payloads Sender Test ===\n');
final uuid = const Uuid();
final correlationId = uuid.v4();
print('Correlation ID: $correlationId');
print('Subject: $TEST_SUBJECT');
print('Broker URL: $TEST_BROKER_URL');
print('Fileserver URL: $TEST_FILESERVER_URL');
print('Size Threshold: $SIZE_THRESHOLD bytes (1MB)\n');
// Create sample data for each type (mirroring Julia test)
final textData = 'Hello! This is a test chat message. 🎉\nHow are you doing today? 😊';
final dictData = {
'type': 'chat',
'sender': 'serviceA',
'receiver': 'serviceB',
'metadata': {
'timestamp': DateTime.now().toUtc().toIsoString(),
'priority': 'high',
'tags': ['urgent', 'chat', 'test']
},
'content': {
'text': 'This is a JSON-formatted chat message with nested structure.',
'format': 'markdown',
'mentions': ['user1', 'user2']
}
};
// Arrow table data (small - direct transport)
final arrowTableSmall = [
{'id': 1, 'name': 'Alice', 'score': 95, 'active': true},
{'id': 2, 'name': 'Bob', 'score': 88, 'active': false},
{'id': 3, 'name': 'Charlie', 'score': 92, 'active': true},
{'id': 4, 'name': 'Diana', 'score': 78, 'active': true},
{'id': 5, 'name': 'Eve', 'score': 85, 'active': false},
{'id': 6, 'name': 'Frank', 'score': 91, 'active': true},
{'id': 7, 'name': 'Grace', 'score': 89, 'active': true},
{'id': 8, 'name': 'Henry', 'score': 76, 'active': false},
{'id': 9, 'name': 'Ivy', 'score': 94, 'active': true},
{'id': 10, 'name': 'Jack', 'score': 82, 'active': true}
];
// Json table data (small - direct transport)
final jsonTableSmall = [
{'id': 1, 'name': 'Alice', 'score': 95, 'active': true},
{'id': 2, 'name': 'Bob', 'score': 88, 'active': false},
{'id': 3, 'name': 'Charlie', 'score': 92, 'active': true},
{'id': 4, 'name': 'Diana', 'score': 78, 'active': true},
{'id': 5, 'name': 'Eve', 'score': 85, 'active': false},
{'id': 6, 'name': 'Frank', 'score': 91, 'active': true},
{'id': 7, 'name': 'Grace', 'score': 89, 'active': true},
{'id': 8, 'name': 'Henry', 'score': 76, 'active': false},
{'id': 9, 'name': 'Ivy', 'score': 94, 'active': true},
{'id': 10, 'name': 'Jack', 'score': 82, 'active': true}
];
// Audio data (small binary - direct transport)
final audioData = Uint8List(100);
for (int i = 0; i < 100; i++) {
audioData[i] = (Random().nextRange(1, 255)).toInt();
}
// Video data (small binary - direct transport)
final videoData = Uint8List(150);
for (int i = 0; i < 150; i++) {
videoData[i] = (Random().nextRange(1, 255)).toInt();
}
// Binary data (small - direct transport)
final binaryData = Uint8List(200);
for (int i = 0; i < 200; i++) {
binaryData[i] = (Random().nextRange(1, 255)).toInt();
}
// Large data for link transport testing
final largeArrowTable = List.generate(20000, (i) => {
'id': i + 1,
'name': 'user_${i + 1}',
'score': (Random().nextRange(50, 100)).toInt(),
'active': Random().nextBool(),
'timestamp': DateTime.now().toUtc().toIsoString()
});
final largeJsonTable = List.generate(50000, (i) => {
'id': i + 1,
'name': 'user_${i + 1}',
'score': (Random().nextRange(50, 100)).toInt(),
'active': Random().nextBool()
});
final largeAudioData = Uint8List(1500000);
for (int i = 0; i < 1500000; i++) {
largeAudioData[i] = (Random().nextRange(1, 255)).toInt();
}
final largeVideoData = Uint8List(1500000);
for (int i = 0; i < 1500000; i++) {
largeVideoData[i] = (Random().nextRange(1, 255)).toInt();
}
final largeBinaryData = Uint8List(1500000);
for (int i = 0; i < 1500000; i++) {
largeBinaryData[i] = (Random().nextRange(1, 255)).toInt();
}
// Read image files from disk (following Julia test pattern)
final filePathSmallImage = './test/small_image.jpg';
final fileDataSmallImage = File(filePathSmallImage).readAsBytesSync();
final filenameSmallImage = filePathSmallImage.split('/').last;
final filePathLargeImage = './test/large_image.png';
final fileDataLargeImage = File(filePathLargeImage).readAsBytesSync();
final filenameLargeImage = filePathLargeImage.split('/').last;
logTrace('Creating payloads list with mixed content');
// Create payloads list - mixed content with both small and large data
// Small data uses direct transport, large data uses link transport
final payloads = [
// Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image
['chat_text', textData, 'text'],
['chat_json', dictData, 'dictionary'],
// ['arrow_table_small', arrowTableSmall, 'arrowtable'],
['json_table_small', jsonTableSmall, 'jsontable'],
[filenameSmallImage, fileDataSmallImage, 'binary'],
// Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary
// ['arrow_table_large', largeArrowTable, 'arrowtable'],
['json_table_large', largeJsonTable, 'jsontable'],
[filenameLargeImage, fileDataLargeImage, 'binary'],
// ['audio_clip_large', largeAudioData, 'audio'],
// ['video_clip_large', largeVideoData, 'video'],
// ['binary_file_large', largeBinaryData, 'binary']
];
logTrace('Total payloads: ${payloads.length}');
try {
// Send the message
print('Sending mixed payloads...\n');
final (env, envJsonStr) = await natsbridge.smartsend(
TEST_SUBJECT,
payloads,
brokerUrl: TEST_BROKER_URL,
fileserverUrl: TEST_FILESERVER_URL,
fileserverUploadHandler: natsbridge.plikOneshotUpload,
sizeThreshold: SIZE_THRESHOLD,
correlationId: correlationId,
msgPurpose: 'chat',
senderName: 'dart-mix-test',
receiverName: '',
receiverId: '',
replyTo: '',
replyToMsgId: '',
isPublish: true,
);
print('\n=== Envelope Created ===');
print('Correlation ID: ${env['correlation_id']}');
print('Message ID: ${env['msg_id']}');
print('Timestamp: ${env['timestamp']}');
print('Subject: ${env['send_to']}');
print('Purpose: ${env['msg_purpose']}');
print('Sender: ${env['sender_name']}');
print('Payloads: ${env['payloads'].length}\n');
// Log transport type for each payload
for (int i = 0; i < env['payloads'].length; i++) {
final payload = env['payloads'][i];
logTrace('Payload ${i + 1} (${payload['dataname']}):');
logTrace(' Transport: ${payload['transport']}');
logTrace(' Type: ${payload['payload_type']}');
logTrace(' Size: ${payload['size']} bytes');
logTrace(' Encoding: ${payload['encoding']}');
if (payload['transport'] == 'link') {
logTrace(' URL: ${payload['data']}');
}
}
// Summary
print('\n--- Transport Summary ---');
final directCount = env['payloads'].where((p) => p['transport'] == 'direct').length;
final linkCount = env['payloads'].where((p) => p['transport'] == 'link').length;
logTrace('Direct transport: $directCount payloads');
logTrace('Link transport: $linkCount payloads');
print('\nTest completed.');
} catch (error) {
print('\n❌ Test failed with error: $error');
print('$error');
exit(1);
}
}
void main() {
runTest();
}