#!/usr/bin/env julia # Test script for mixed-content message testing # Tests receiving a mix of text, json, table, image, audio, video, and binary data # from Julia serviceA to Julia serviceB using msghandler.jl smartunpack # # This test demonstrates that any combination and any number of mixed content # can be sent and received correctly. using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64 # Include the bridge module include("./src/msghandler.jl") using .msghandler # Configuration const SUBJECT = "/test/mix" const NATS_URL = "nats.yiem.cc" const FILESERVER_URL = "http://192.168.88.104:8080" # ------------------------------------------------------------------------------------------------ # # test mixed content transfer # # ------------------------------------------------------------------------------------------------ # # Helper: Log with correlation ID function log_trace(message) timestamp = Dates.now() println("[$timestamp] $message") end # Receiver: Listen for messages and verify mixed content handling function test_mix_receive() conn = NATS.connect(NATS_URL) incoming_msg = nothing NATS.subscribe(conn, SUBJECT) do msg log_trace("Received message on $(msg.subject)") incoming_msg = msg # # Use msghandler.smartunpack to handle the data # # API: smartunpack(msg, download_handler; max_retries, base_delay, max_delay) # result = msghandler.smartunpack( # msg; # max_retries = 5, # base_delay = 100, # max_delay = 5000 # ) # log_trace("Received $(length(result["payloads"])) payloads") # # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples # for (dataname, data, data_type) in result["payloads"] # log_trace("\n=== Payload: $dataname (type: $data_type) ===") # # Handle different data types # if data_type == "text" # # Text data - should be a String # if isa(data, String) # log_trace(" Type: String") # log_trace(" Length: $(length(data)) characters") # # Display first 200 characters # if length(data) > 200 # log_trace(" First 200 chars: $(data[1:200])...") # else # log_trace(" Content: $data") # end # # Save to file # output_path = "./received_$dataname.txt" # write(output_path, data) # log_trace(" Saved to: $output_path") # else # log_trace(" ERROR: Expected String, got $(typeof(data))") # end # elseif data_type == "dictionary" # # Dictionary data - should be JSON object # if isa(data, JSON.Object{String, Any}) # log_trace(" Type: Dict") # log_trace(" Keys: $(keys(data))") # # Display nested content # for (key, value) in data # log_trace(" $key => $value") # end # # Save to JSON file # output_path = "./received_$dataname.json" # json_str = JSON.json(data, 2) # write(output_path, json_str) # log_trace(" Saved to: $output_path") # else # log_trace(" ERROR: Expected Dict, got $(typeof(data))") # end # elseif data_type == "table" # # Table data - should be a DataFrame # tabledata = deepcopy(data) # println("found table data") # break # # return data # # if isa(data, DataFrame) # # log_trace(" Type: DataFrame") # # log_trace(" Dimensions: $(size(data, 1)) rows x $(size(data, 2)) columns") # # log_trace(" Columns: $(names(data))") # # # Display first few rows # # log_trace(" First 5 rows:") # # display(data[1:min(5, size(data, 1)), :]) # # # Save to Arrow file # # output_path = "./received_$dataname.arrow" # # io = IOBuffer() # # Arrow.write(io, data) # # write(output_path, take!(io)) # # log_trace(" Saved to: $output_path") # # else # # log_trace(" ERROR: Expected DataFrame, got $(typeof(data))") # # end # elseif data_type == "image" # # Image data - should be Vector{UInt8} # if isa(data, Vector{UInt8}) # log_trace(" Type: Vector{UInt8} (binary)") # log_trace(" Size: $(length(data)) bytes") # # Save to file # output_path = "./received_$dataname.bin" # write(output_path, data) # log_trace(" Saved to: $output_path") # else # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") # end # elseif data_type == "audio" # # Audio data - should be Vector{UInt8} # if isa(data, Vector{UInt8}) # log_trace(" Type: Vector{UInt8} (binary)") # log_trace(" Size: $(length(data)) bytes") # # Save to file # output_path = "./received_$dataname.bin" # write(output_path, data) # log_trace(" Saved to: $output_path") # else # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") # end # elseif data_type == "video" # # Video data - should be Vector{UInt8} # if isa(data, Vector{UInt8}) # log_trace(" Type: Vector{UInt8} (binary)") # log_trace(" Size: $(length(data)) bytes") # # Save to file # output_path = "./received_$dataname.bin" # write(output_path, data) # log_trace(" Saved to: $output_path") # else # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") # end # elseif data_type == "binary" # # Binary data - should be Vector{UInt8} # if isa(data, Vector{UInt8}) # log_trace(" Type: Vector{UInt8} (binary)") # log_trace(" Size: $(length(data)) bytes") # # Save to file # output_path = "./received_$dataname.bin" # write(output_path, data) # log_trace(" Saved to: $output_path") # else # log_trace(" ERROR: Expected Vector{UInt8}, got $(typeof(data))") # end # else # log_trace(" ERROR: Unknown data type '$data_type'") # end # end # Summary # println("\n=== Verification Summary ===") # text_count = count(x -> x[3] == "text", result["payloads"]) # dict_count = count(x -> x[3] == "dictionary", result["payloads"]) # table_count = count(x -> x[3] == "table", result["payloads"]) # image_count = count(x -> x[3] == "image", result["payloads"]) # audio_count = count(x -> x[3] == "audio", result["payloads"]) # video_count = count(x -> x[3] == "video", result["payloads"]) # binary_count = count(x -> x[3] == "binary", result["payloads"]) # log_trace("Text payloads: $text_count") # log_trace("Dictionary payloads: $dict_count") # log_trace("Table payloads: $table_count") # log_trace("Image payloads: $image_count") # log_trace("Audio payloads: $audio_count") # log_trace("Video payloads: $video_count") # log_trace("Binary payloads: $binary_count") # # Print transport type info for each payload if available # println("\n=== Payload Details ===") # for (dataname, data, data_type) in result["payloads"] # if data_type in ["image", "audio", "video", "binary"] # log_trace("$dataname: $(length(data)) bytes (binary)") # elseif data_type == "table" # data = DataFrame(data) # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)") # elseif data_type == "dictionary" # log_trace("$dataname: $(length(JSON.json(data))) bytes (Dict)") # elseif data_type == "text" # log_trace("$dataname: $(length(data)) characters (String)") # end # end end # Keep listening for 2 minutes sleep(20) NATS.drain(conn) return incoming_msg end # Run the test println("Starting mixed-content transport test...") println("Note: This receiver will wait for messages from the sender.") println("Run test_julia_to_julia_mix_sender.jl first to send test data.") # Run receiver println("\ntesting smartunpack for mixed content") incoming_msg = test_mix_receive() println("\nTest completed.") Check architecture.md. For sending table I want to add JSON in addition to Apache Arrow. Currently I use "table" datatype when sending table data using Arrow. Now table that I want to send using JSON I will use "jsontable" as datatype while sending table using Arrow I will use "arrowtable" as datatype. This will select how smartpack and smartunpack serialize/deserialize the table. Can you help me do this? Save the updated architecture.md into updated_architecture.md file. I will deal with source code later. Now update implementation.md and save into updated_implementation.md Keep in mind that Julia DataFrame and Python Pandas rely on columnar-oriented dictionary to create as the following example: julia> dict = Dict("customer age" => [15, 20, 25], "first name" => ["Rohit", "Rahul", "Akshat"]) julia> DataFrame(dict) python> data = { "Name": ["Alice", "Bob", "Charlie"], "Age": [25, 30, 35], "Score": [88.5, 92.0, 79.5] } python> df = pd.DataFrame(data) But JS use Array of Objects while MicroPython use list of lists. Both are row-oriented structure. So use row-oriented JSON to send across these languages. For Julia and Python, only convert row-oriented JSON to columnar-oriented dictionary for "going-into" and vise versa for "coming-out" a dataframe while JS and MicroPython won't require such process. You may add these info into architecture.md if you see fit.