From f534248becf9b55e495c872aa06fd105c4baeb33 Mon Sep 17 00:00:00 2001 From: narawat Date: Sun, 8 Mar 2026 10:42:54 +0700 Subject: [PATCH] update --- docs/architecture.md | 2 +- etc.jl | 0 etc.txt | 310 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 311 insertions(+), 1 deletion(-) delete mode 100644 etc.jl create mode 100644 etc.txt diff --git a/docs/architecture.md b/docs/architecture.md index 500f988..d496e92 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ end |---------|---------| | `NATS.jl` | Core NATS functionality | | `Arrow.jl` | Arrow IPC serialization | -| `JSON3.jl` | JSON parsing | +| `JSON.jl` | JSON parsing | | `HTTP.jl` | HTTP client for file server | | `UUIDs.jl` | UUID generation | | `Dates.jl` | Timestamps | diff --git a/etc.jl b/etc.jl deleted file mode 100644 index e69de29..0000000 diff --git a/etc.txt b/etc.txt new file mode 100644 index 0000000..074b9bc --- /dev/null +++ b/etc.txt @@ -0,0 +1,310 @@ +#!/usr/bin/env julia +# Test script for mixed-content message testing +# Tests receiving a mix of text, json, table, image, audio, video, and binary data +# from Julia serviceA to Julia serviceB using NATSBridge.jl smartreceive +# +# This test demonstrates that any combination and any number of mixed content +# can be sent and received correctly. + +using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64 + +# Include the bridge module +include("./src/NATSBridge.jl") +using .NATSBridge + +# Configuration +const SUBJECT = "/test/mix" +const NATS_URL = "nats.yiem.cc" +const FILESERVER_URL = "http://192.168.88.104:8080" + + +# ------------------------------------------------------------------------------------------------ # +# test mixed content transfer # +# ------------------------------------------------------------------------------------------------ # + + +# Helper: Log with correlation ID +function log_trace(message) + timestamp = Dates.now() + println("[$timestamp] $message") +end + + +# Receiver: Listen for messages and verify mixed content handling +function test_mix_receive() + conn = NATS.connect(NATS_URL) + incoming_msg = nothing + NATS.subscribe(conn, SUBJECT) do msg + log_trace("Received message on $(msg.subject)") + incoming_msg = msg + + # # Use NATSBridge.smartreceive to handle the data + # # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay) + # result = NATSBridge.smartreceive( + # msg; + # max_retries = 5, + # base_delay = 100, + # max_delay = 5000 + # ) + + # log_trace("Received $(length(result["payloads"])) payloads") + + + # # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples + # for (dataname, data, data_type) in result["payloads"] + # log_trace("\n=== Payload: $dataname (type: $data_type) ===") + + # # Handle different data types + # if data_type == "text" + # # Text data - should be a String + # if isa(data, String) + # log_trace(" Type: String") + # log_trace(" Length: $(length(data)) characters") + + # # Display first 200 characters + # if length(data) > 200 + # log_trace(" First 200 chars: $(data[1:200])...") + # else + # log_trace(" Content: $data") + # end + + # # Save to file + # output_path = "./received_$dataname.txt" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected String, got $(typeof(data))") + # end + + # elseif data_type == "dictionary" + # # Dictionary data - should be JSON object + # if isa(data, JSON.Object{String, Any}) + # log_trace(" Type: Dict") + # log_trace(" Keys: $(keys(data))") + + # # Display nested content + # for (key, value) in data + # log_trace(" $key => $value") + # end + + # # Save to JSON file + # output_path = "./received_$dataname.json" + # json_str = JSON.json(data, 2) + # write(output_path, json_str) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Dict, got $(typeof(data))") + # end + + # elseif data_type == "table" + # # Table data - should be a DataFrame + # tabledata = deepcopy(data) + # println("found table data") + # break + # # return data + # # if isa(data, DataFrame) + # # log_trace(" Type: DataFrame") + # # log_trace(" Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns") + # # log_trace(" Columns: $(names(data))") + + # # # Display first few rows + # # log_trace(" First 5 rows:") + # # display(data[1:min(5, size(data, 1)), :]) + + # # # Save to Arrow file + # # output_path = "./received_$dataname.arrow" + # # io = IOBuffer() + # # Arrow.write(io, data) + # # write(output_path, take!(io)) + # # log_trace(" Saved to: $output_path") + # # else + # # log_trace(" ERROR: Expected DataFrame, got $(typeof(data))") + # # end + + # elseif data_type == "image" + # # Image data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # elseif data_type == "audio" + # # Audio data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # elseif data_type == "video" + # # Video data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # elseif data_type == "binary" + # # Binary data - should be Vector{UInt8} + # if isa(data, Vector{UInt8}) + # log_trace(" Type: Vector{UInt8} (binary)") + # log_trace(" Size: $(length(data)) bytes") + + # # Save to file + # output_path = "./received_$dataname.bin" + # write(output_path, data) + # log_trace(" Saved to: $output_path") + # else + # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") + # end + + # else + # log_trace(" ERROR: Unknown data type '$data_type'") + # end + # end + + # Summary + # println("\n=== Verification Summary ===") + # text_count = count(x -> x[3] == "text", result["payloads"]) + # dict_count = count(x -> x[3] == "dictionary", result["payloads"]) + # table_count = count(x -> x[3] == "table", result["payloads"]) + # image_count = count(x -> x[3] == "image", result["payloads"]) + # audio_count = count(x -> x[3] == "audio", result["payloads"]) + # video_count = count(x -> x[3] == "video", result["payloads"]) + # binary_count = count(x -> x[3] == "binary", result["payloads"]) + + # log_trace("Text payloads: $text_count") + # log_trace("Dictionary payloads: $dict_count") + # log_trace("Table payloads: $table_count") + # log_trace("Image payloads: $image_count") + # log_trace("Audio payloads: $audio_count") + # log_trace("Video payloads: $video_count") + # log_trace("Binary payloads: $binary_count") + + # # Print transport type info for each payload if available + # println("\n=== Payload Details ===") + # for (dataname, data, data_type) in result["payloads"] + # if data_type in ["image", "audio", "video", "binary"] + # log_trace("$dataname: $(length(data)) bytes (binary)") + # elseif data_type == "table" + # data = DataFrame(data) + # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)") + # elseif data_type == "dictionary" + # log_trace("$dataname: $(length(JSON.json(data))) bytes (Dict)") + # elseif data_type == "text" + # log_trace("$dataname: $(length(data)) characters (String)") + # end + # end + end + + # Keep listening for 2 minutes + sleep(20) + NATS.drain(conn) + return incoming_msg +end + + +# Run the test +println("Starting mixed-content transport test...") +println("Note: This receiver will wait for messages from the sender.") +println("Run test_julia_to_julia_mix_sender.jl first to send test data.") + +# Run receiver +println("\ntesting smartreceive for mixed content") +incoming_msg = test_mix_receive() + +println("\nTest completed.") + + + + + + + + + + + + + + +Check architecture.md. For sending table I want to add JSON in addition to Apache Arrow. +Currently I use "table" datatype when sending table data using Arrow. Now table that I want to send using JSON +I will use "jsontable" as datatype while sending table using Arrow I will use "arrowtable" as datatype. +This will select how smartsend and smartreceive serialize/deserialize the table. + +Can you help me do this? Save the updated architecture.md into updated_architecture.md file. I will deal with source code later. + + + + +Now update implementation.md and save into updated_implementation.md +Keep in mind that Julia DataFrame and Python Pandas rely on columnar-oriented dictionary to create as the following example: +julia> dict = Dict("customer age" => [15, 20, 25], + "first name" => ["Rohit", "Rahul", "Akshat"]) +julia> DataFrame(dict) + +python> data = { + "Name": ["Alice", "Bob", "Charlie"], + "Age": [25, 30, 35], + "Score": [88.5, 92.0, 79.5] +} + +python> df = pd.DataFrame(data) + + +But JS use Array of Objects while MicroPython use list of lists. Both are row-oriented structure. +So use row-oriented JSON to send across these languages. For Julia and Python, only convert +row-oriented JSON to columnar-oriented dictionary for "going-into" and vise versa for "coming-out" +a dataframe while JS and MicroPython won't require such process. +You may add these info into architecture.md if you see fit. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +