1st commit

test/test_js_mix_payloads_receiver.js

@@ -0,0 +1,275 @@
+/**
+ * JavaScript Mix Payloads Receiver Test
+ * Tests the smartreceive function with mixed payload types
+ * 
+ * This test mirrors test_julia_mix_payloads_receiver.jl and demonstrates that
+ * any combination and any number of mixed content can be received correctly.
+ */
+
+const msghandler = require('../src/msghandler.js');
+const nats = require('nats');
+const crypto = require('crypto');
+
+const TEST_SUBJECT = '/msghandler';
+const TEST_BROKER_URL = process.env.NATS_URL || 'nats.yiem.cc';
+const TEST_FILESERVER_URL = process.env.FILESERVER_URL || 'http://192.168.88.104:8080';
+
+async function runTest() {
+    console.log('=== JavaScript Mix Payloads Receiver Test ===\n');
+
+    const correlationId = crypto.randomUUID();
+    console.log(`Correlation ID: ${correlationId}`);
+    console.log(`Subject: ${TEST_SUBJECT}`);
+    console.log(`Broker URL: ${TEST_BROKER_URL}`);
+    console.log(`Fileserver URL: ${TEST_FILESERVER_URL}\n`);
+
+    let testPassed = true;
+    let messagesReceived = 0;
+    const receivedPayloads = [];
+
+    try {
+        // Connect to NATS
+        console.log('Connecting to NATS server...');
+        const nc = await nats.connect({ servers: TEST_BROKER_URL });
+        console.log('✅ Connected to NATS server\n');
+
+        // Set up message subscription
+        const subscription = nc.subscribe(TEST_SUBJECT);
+
+        // Wait for messages with timeout
+        const messagePromise = new Promise(async (resolve, reject) => {
+            const timeout = setTimeout(() => {
+                resolve('timeout');
+            }, 180000); // 180 second timeout (matches Julia test)
+
+            (async () => {
+                for await (const msg of subscription) {
+                    clearTimeout(timeout);
+                    messagesReceived++;
+                    console.log(`\n=== Message ${messagesReceived} Received ===`);
+                    console.log(`Received message on ${msg.subject}`);
+
+                    try {
+                        // Process the message using smartreceive
+                        const envelope = await msghandler.smartreceive(msg, {
+                            fileserver_download_handler: msghandler.fetchWithBackoff,
+                            max_retries: 5,
+                            base_delay: 100,
+                            max_delay: 5000
+                        });
+
+                        console.log(`Correlation ID: ${envelope.correlation_id}`);
+                        console.log(`Message ID: ${envelope.msg_id}`);
+                        console.log(`Timestamp: ${envelope.timestamp}`);
+                        console.log(`Purpose: ${envelope.msg_purpose}`);
+                        console.log(`Sender: ${envelope.sender_name}`);
+                        console.log(`Number of payloads: ${envelope.payloads.length}`);
+
+                        receivedPayloads.push(envelope);
+
+                        // Validate envelope structure
+                        console.log('\n=== Envelope Validation ===');
+
+                        if (envelope.payloads.length < 1) {
+                            console.log(`❌ Expected at least 1 payload, got ${envelope.payloads.length}`);
+                            testPassed = false;
+                        } else {
+                            console.log(`✅ Correct number of payloads: ${envelope.payloads.length}`);
+                        }
+
+                        // Process all payloads in the envelope
+                        console.log('\n=== Processing Payloads ===');
+                        for (let i = 0; i < envelope.payloads.length; i++) {
+                            const [dataname, data, dataType] = envelope.payloads[i];
+
+                            console.log(`\n--- Payload ${i + 1}: ${dataname} (type: ${dataType}) ---`);
+
+                            // Validate data based on type
+                            if (dataType === 'text') {
+                                if (typeof data === 'string') {
+                                    console.log(`✅ Text data received (${data.length} chars)`);
+                                    console.log(`   First 200 chars: "${data.substring(0, 200)}${data.length > 200 ? '...' : ''}"`);
+                                    
+                                    // Save to file
+                                    const outputPath = `./received_${dataname}.txt`;
+                                    require('fs').writeFileSync(outputPath, data);
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ Text data is not a string, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'dictionary') {
+                                if (typeof data === 'object' && data !== null && !Array.isArray(data)) {
+                                    console.log(`✅ Dictionary data received`);
+                                    console.log(`   Keys: ${Object.keys(data).join(', ')}`);
+                                    
+                                    // Save to JSON file
+                                    const outputPath = `./received_${dataname}.json`;
+                                    require('fs').writeFileSync(outputPath, JSON.stringify(data, null, 2));
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ Dictionary data is not an object, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'arrowtable') {
+                                // Arrow tables have numRows and numCols properties
+                                if (data && typeof data === 'object' && 
+                                    (data.numRows !== undefined || data.numRows !== null) &&
+                                    (data.numCols !== undefined || data.numCols !== null)) {
+                                    console.log(`✅ Arrow table data received`);
+                                    console.log(`   Rows: ${data.numRows}, Columns: ${data.numCols}`);
+                                    
+                                    // Save to file
+                                    const outputPath = `./received_${dataname}.arrow`;
+                                    // Note: Actual Arrow IPC serialization would require apache-arrow library
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else if (data && typeof data === 'object') {
+                                    // Some Arrow implementations may have different properties
+                                    console.log(`✅ Arrow table data received (non-standard format)`);
+                                    console.log(`   Keys: ${Object.keys(data).join(', ')}`);
+                                } else {
+                                    console.log(`❌ Arrow table data is not a valid object, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'jsontable') {
+                                if (Array.isArray(data)) {
+                                    console.log(`✅ JSON table data received`);
+                                    console.log(`   Rows: ${data.length}`);
+                                    if (data.length > 0) {
+                                        console.log(`   Columns: ${Object.keys(data[0]).join(', ')}`);
+                                    }
+                                    
+                                    // Save to JSON file
+                                    const outputPath = `./received_${dataname}.json`;
+                                    require('fs').writeFileSync(outputPath, JSON.stringify(data, null, 2));
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ JSON table data is not an array, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'image') {
+                                if (data instanceof Buffer || data instanceof Uint8Array) {
+                                    const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data);
+                                    console.log(`✅ Image data received (${dataBuffer.length} bytes)`);
+                                    
+                                    // Save to file
+                                    const outputPath = `./received_${dataname}.bin`;
+                                    require('fs').writeFileSync(outputPath, dataBuffer);
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ Image data is not a Buffer or Uint8Array, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'audio') {
+                                if (data instanceof Buffer || data instanceof Uint8Array) {
+                                    const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data);
+                                    console.log(`✅ Audio data received (${dataBuffer.length} bytes)`);
+                                    
+                                    // Save to file
+                                    const outputPath = `./received_${dataname}.bin`;
+                                    require('fs').writeFileSync(outputPath, dataBuffer);
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ Audio data is not a Buffer or Uint8Array, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'video') {
+                                if (data instanceof Buffer || data instanceof Uint8Array) {
+                                    const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data);
+                                    console.log(`✅ Video data received (${dataBuffer.length} bytes)`);
+                                    
+                                    // Save to file
+                                    const outputPath = `./received_${dataname}.bin`;
+                                    require('fs').writeFileSync(outputPath, dataBuffer);
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ Video data is not a Buffer or Uint8Array, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else if (dataType === 'binary') {
+                                if (data instanceof Buffer || data instanceof Uint8Array) {
+                                    const dataBuffer = Buffer.isBuffer(data) ? data : Buffer.from(data);
+                                    console.log(`✅ Binary data received (${dataBuffer.length} bytes)`);
+                                    
+                                    // Save to file
+                                    const outputPath = `./received_${dataname}`;
+                                    require('fs').writeFileSync(outputPath, dataBuffer);
+                                    console.log(`   Saved to: ${outputPath}`);
+                                } else {
+                                    console.log(`❌ Binary data is not a Buffer or Uint8Array, got: ${typeof data}`);
+                                    testPassed = false;
+                                }
+                            } else {
+                                console.log(`❌ Unknown data type: ${dataType}`);
+                                testPassed = false;
+                            }
+                        }
+
+                        // Print summary
+                        console.log('\n=== Verification Summary ===');
+                        const textCount = envelope.payloads.filter(p => p[2] === 'text').length;
+                        const dictCount = envelope.payloads.filter(p => p[2] === 'dictionary').length;
+                        const arrowtableCount = envelope.payloads.filter(p => p[2] === 'arrowtable').length;
+                        const jsontableCount = envelope.payloads.filter(p => p[2] === 'jsontable').length;
+                        const imageCount = envelope.payloads.filter(p => p[2] === 'image').length;
+                        const audioCount = envelope.payloads.filter(p => p[2] === 'audio').length;
+                        const videoCount = envelope.payloads.filter(p => p[2] === 'video').length;
+                        const binaryCount = envelope.payloads.filter(p => p[2] === 'binary').length;
+
+                        console.log(`Text payloads: ${textCount}`);
+                        console.log(`Dictionary payloads: ${dictCount}`);
+                        console.log(`Arrow table payloads: ${arrowtableCount}`);
+                        console.log(`JSON table payloads: ${jsontableCount}`);
+                        console.log(`Image payloads: ${imageCount}`);
+                        console.log(`Audio payloads: ${audioCount}`);
+                        console.log(`Video payloads: ${videoCount}`);
+                        console.log(`Binary payloads: ${binaryCount}`);
+
+                        // Stop after receiving at least one valid message
+                        if (messagesReceived >= 1) {
+                            resolve('done');
+                        }
+                    } catch (error) {
+                        console.error(`❌ Error processing message: ${error.message}`);
+                        console.error(error.stack);
+                        testPassed = false;
+                        resolve('error');
+                    }
+                }
+            })();
+        });
+
+        console.log('Waiting for messages...\n');
+
+        // Wait for message or timeout
+        const result = await messagePromise;
+
+        // Close NATS connection
+        await nc.close();
+        console.log('\n✅ NATS connection closed');
+
+        // Final result
+        console.log('\n=== Test Result ===');
+        if (messagesReceived === 0) {
+            console.log('❌ NO MESSAGES RECEIVED');
+            console.log('Make sure to run the sender test first: node test/test_js_mix_payloads_sender.js');
+            process.exit(1);
+        } else if (result === 'error') {
+            console.log('❌ ERROR PROCESSING MESSAGES');
+            process.exit(1);
+        } else if (testPassed) {
+            console.log('✅ ALL TESTS PASSED');
+            process.exit(0);
+        } else {
+            console.log('❌ SOME TESTS FAILED');
+            process.exit(1);
+        }
+
+    } catch (error) {
+        console.error('❌ Test failed with error:', error.message);
+        console.error(error.stack);
+        process.exit(1);
+    }
+}
+
+runTest();

test/test_js_mix_payloads_sender.js

@@ -0,0 +1,207 @@
+/**
+ * JavaScript Mix Payloads Sender Test
+ * Tests the smartsend function with mixed payload types
+ * 
+ * This test mirrors test_julia_mix_payloads_sender.jl and demonstrates that
+ * any combination and any number of mixed content can be sent correctly.
+ */
+
+const msghandler = require('../src/msghandler.js');
+const crypto = require('crypto');
+const fs = require('fs');
+const path = require('path');
+
+const TEST_SUBJECT = '/msghandler';
+const TEST_BROKER_URL = process.env.NATS_URL || 'nats.yiem.cc';
+const TEST_FILESERVER_URL = process.env.FILESERVER_URL || 'http://192.168.88.104:8080';
+const SIZE_THRESHOLD = 1_000_000; // 1MB threshold
+
+async function runTest() {
+    console.log('=== JavaScript Mix Payloads Sender Test ===\n');
+    
+    const correlationId = crypto.randomUUID();
+    console.log(`Correlation ID: ${correlationId}`);
+    console.log(`Subject: ${TEST_SUBJECT}`);
+    console.log(`Broker URL: ${TEST_BROKER_URL}`);
+    console.log(`Fileserver URL: ${TEST_FILESERVER_URL}`);
+    console.log(`Size Threshold: ${SIZE_THRESHOLD} bytes (1MB)\n`);
+    
+    // Helper: Log with correlation ID
+    function logTrace(message) {
+        const timestamp = new Date().toISOString();
+        console.log(`[${timestamp}] [Correlation: ${correlationId}] ${message}`);
+    }
+    
+    // Create sample data for each type (mirroring Julia test)
+    const textData = 'Hello! This is a test chat message. 🎉\nHow are you doing today? 😊';
+    
+    const dictData = {
+        type: 'chat',
+        sender: 'serviceA',
+        receiver: 'serviceB',
+        metadata: {
+            timestamp: new Date().toISOString(),
+            priority: 'high',
+            tags: ['urgent', 'chat', 'test']
+        },
+        content: {
+            text: 'This is a JSON-formatted chat message with nested structure.',
+            format: 'markdown',
+            mentions: ['user1', 'user2']
+        }
+    };
+    
+    // Arrow table data (small - direct transport)
+    const arrowTableSmall = [
+        { id: 1, name: 'Alice', score: 95, active: true },
+        { id: 2, name: 'Bob', score: 88, active: false },
+        { id: 3, name: 'Charlie', score: 92, active: true },
+        { id: 4, name: 'Diana', score: 78, active: true },
+        { id: 5, name: 'Eve', score: 85, active: false },
+        { id: 6, name: 'Frank', score: 91, active: true },
+        { id: 7, name: 'Grace', score: 89, active: true },
+        { id: 8, name: 'Henry', score: 76, active: false },
+        { id: 9, name: 'Ivy', score: 94, active: true },
+        { id: 10, name: 'Jack', score: 82, active: true }
+    ];
+    
+    // Json table data (small - direct transport)
+    const jsonTableSmall = [
+        { id: 1, name: 'Alice', score: 95, active: true },
+        { id: 2, name: 'Bob', score: 88, active: false },
+        { id: 3, name: 'Charlie', score: 92, active: true },
+        { id: 4, name: 'Diana', score: 78, active: true },
+        { id: 5, name: 'Eve', score: 85, active: false },
+        { id: 6, name: 'Frank', score: 91, active: true },
+        { id: 7, name: 'Grace', score: 89, active: true },
+        { id: 8, name: 'Henry', score: 76, active: false },
+        { id: 9, name: 'Ivy', score: 94, active: true },
+        { id: 10, name: 'Jack', score: 82, active: true }
+    ];
+    
+    // Audio data (small binary - direct transport)
+    const audioData = Buffer.alloc(100);
+    for (let i = 0; i < 100; i++) {
+        audioData[i] = Math.floor(Math.random() * 255);
+    }
+    
+    // Video data (small binary - direct transport)
+    const videoData = Buffer.alloc(150);
+    for (let i = 0; i < 150; i++) {
+        videoData[i] = Math.floor(Math.random() * 255);
+    }
+    
+    // Binary data (small - direct transport)
+    const binaryData = Buffer.alloc(200);
+    for (let i = 0; i < 200; i++) {
+        binaryData[i] = Math.floor(Math.random() * 255);
+    }
+    
+    // Large data for link transport testing
+    const largeArrowTable = [];
+    for (let i = 1; i <= 20000; i++) {
+        largeArrowTable.push({
+            id: i,
+            name: `user_${i}`,
+            score: Math.floor(Math.random() * 51) + 50,
+            active: Math.random() > 0.5,
+            timestamp: new Date().toISOString()
+        });
+    }
+    
+    const largeJsonTable = [];
+    for (let i = 1; i <= 50000; i++) {
+        largeJsonTable.push({
+            id: i,
+            name: `user_${i}`,
+            score: Math.floor(Math.random() * 51) + 50,
+            active: Math.random() > 0.5
+        });
+    }
+    
+    const largeAudioData = Buffer.alloc(1_500_000);
+    for (let i = 0; i < 1_500_000; i++) {
+        largeAudioData[i] = Math.floor(Math.random() * 255);
+    }
+    
+    const largeVideoData = Buffer.alloc(1_500_000);
+    for (let i = 0; i < 1_500_000; i++) {
+        largeVideoData[i] = Math.floor(Math.random() * 255);
+    }
+    
+    const largeBinaryData = Buffer.alloc(1_500_000);
+    for (let i = 0; i < 1_500_000; i++) {
+        largeBinaryData[i] = Math.floor(Math.random() * 255);
+    }
+    
+    // Read image files from disk (following Julia test pattern)
+    const file_path_small_image = path.join(__dirname, 'small_image.jpg');
+    const file_data_small_image = fs.readFileSync(file_path_small_image);
+    const filename_small_image = path.basename(file_path_small_image);
+    
+    const file_path_large_image = path.join(__dirname, 'large_image.png');
+    const file_data_large_image = fs.readFileSync(file_path_large_image);
+    const filename_large_image = path.basename(file_path_large_image);
+    
+    logTrace('Creating payloads list with mixed content');
+    
+    // Create payloads list - mixed content with both small and large data
+    // Small data uses direct transport, large data uses link transport
+    const payloads = [
+        // Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image
+        ['chat_text', textData, 'text'],
+        ['chat_json', dictData, 'dictionary'],
+        // ['arrow_table_small', arrowTableSmall, 'arrowtable'],
+        ['json_table_small', jsonTableSmall, 'jsontable'],
+        [filename_small_image, file_data_small_image, 'binary'],
+        
+        // Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary
+        // ['arrow_table_large', largeArrowTable, 'arrowtable'],
+        ['json_table_large', largeJsonTable, 'jsontable'],
+        [filename_large_image, file_data_large_image, 'binary'],
+        // ['audio_clip_large', largeAudioData, 'audio'],
+        // ['video_clip_large', largeVideoData, 'video'],
+        // ['binary_file_large', largeBinaryData, 'binary']
+    ];
+    
+    logTrace(`Total payloads: ${payloads.length}`);
+    
+    try {
+        // Send the message
+        console.log('Sending mixed payloads...\n');
+        const [env, envJsonStr] = await msghandler.smartsend(
+            TEST_SUBJECT,
+            payloads,
+            {
+                broker_url: TEST_BROKER_URL,
+                fileserver_url: TEST_FILESERVER_URL,
+                fileserver_upload_handler: msghandler.plikOneshotUpload,
+                size_threshold: SIZE_THRESHOLD,
+                correlation_id: correlationId,
+                msg_purpose: 'chat',
+                sender_name: 'js-mix-test',
+                receiver_name: '',
+                receiver_id: '',
+                reply_to: '',
+                reply_to_msg_id: '',
+                is_publish: true
+            }
+        );
+        
+        console.log('\n=== Envelope Created ===');
+        console.log(`Correlation ID: ${env.correlation_id}`);
+        console.log(`Message ID: ${env.msg_id}`);
+        console.log(`Timestamp: ${env.timestamp}`);
+        console.log(`Subject: ${env.send_to}`);
+        console.log(`Purpose: ${env.msg_purpose}`);
+        console.log(`Sender: ${env.sender_name}`);
+        console.log(`Payloads: ${env.payloads.length}\n`);
+        
+    } catch (error) {
+        console.error('\n❌ Test failed with error:', error.message);
+        console.error(error.stack);
+        process.exit(1);
+    }
+}
+
+runTest();

test/test_julia_mix_payloads_receiver.jl

@@ -0,0 +1,251 @@
+#!/usr/bin/env julia
+# Test script for mixed-content message testing
+# Tests receiving a mix of text, json, table, image, audio, video, and binary data
+# from Julia serviceA to Julia serviceB using msghandler.jl smartreceive
+#
+# This test demonstrates that any combination and any number of mixed content
+# can be sent and received correctly.
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64
+
+# Include the bridge module
+include("../src/msghandler.jl")
+using .msghandler
+
+# Configuration
+const SUBJECT = "/msghandler"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                      test mixed content transfer                                 #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] $message")
+end
+
+
+# Receiver: Listen for messages and verify mixed content handling
+function test_mix_receive()
+    conn = NATS.connect(NATS_URL)
+    NATS.subscribe(conn, SUBJECT) do msg
+        log_trace("Received message on $(msg.subject)")
+        
+        # Use msghandler.smartreceive to handle the data
+        # API: smartreceive(msg, download_handler; max_retries, base_delay, max_delay)
+        result = msghandler.smartreceive(
+            msg;
+            max_retries = 5,
+            base_delay = 100,
+            max_delay = 5000
+        )
+        
+        log_trace("Received $(length(result["payloads"])) payloads")
+        
+        # Result is an envelope dictionary with payloads field containing list of (dataname, data, data_type) tuples
+        for (dataname, data, data_type) in result["payloads"]
+            log_trace("\n=== Payload: $dataname (type: $data_type) ===")
+            
+            # Handle different data types
+            if data_type == "text"
+                # Text data - should be a String
+                if isa(data, String)
+                    log_trace("  Type: String")
+                    log_trace("  Length: $(length(data)) characters")
+                    
+                    # Display first 200 characters
+                    if length(data) > 200
+                        log_trace("  First 200 chars: $(data[1:200])...")
+                    else
+                        log_trace("  Content: $data")
+                    end
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.txt"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected String, got $(typeof(data))")
+                end
+                
+            elseif data_type == "dictionary"
+                # Dictionary data - should be JSON object
+                if isa(data, JSON.Object{String, Any})
+                    log_trace("  Type: Dict")
+                    log_trace("  Keys: $(keys(data))")
+                    
+                    # Display nested content
+                    for (key, value) in data
+                        log_trace("  $key => $value")
+                    end
+                    
+                    # Save to JSON file
+                    output_path = "./received_$dataname.json"
+                    json_str = JSON.json(data, 2)
+                    write(output_path, json_str)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Dict, got $(typeof(data))")
+                end
+                
+            elseif data_type == "arrowtable"
+                # Arrow table data - should be Arrow.Table
+                if isa(data, Arrow.Table)
+                    log_trace("  Type: Arrow.Table")
+                    
+                    # Convert to DataFrame for display and save
+                    df = DataFrame(data)
+                    @show df[1:3, :]
+                    output_path = "./received_$dataname.arrow"
+                    io = IOBuffer()
+                    Arrow.write(io, data)
+                    write(output_path, take!(io))
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Arrow.Table, got $(typeof(data))")
+                end
+                
+            elseif data_type == "jsontable"
+                # JSON table data - should be Vector{Dict} or Vector{NamedTuple}
+                @show "jsontable" typeof(data)
+                if isa(data, Vector{Any})
+                    log_trace("  Type: Vector{Dict/NamedTuple}")
+                    
+                    # Convert to DataFrame for display and save
+                    df = DataFrame(data)
+                    @show df[1:3, :]
+                    log_trace("  Converted to DataFrame: $(size(df, 1)) rows x $(size(df, 2)) columns")
+                    
+                    # Save as JSON file
+                    output_path = "./received_$dataname.json"
+                    json_str = JSON.json(data, 2)
+                    write(output_path, json_str)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{Dict/NamedTuple}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "image"
+                # Image data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "audio"
+                # Audio data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "video"
+                # Video data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname.bin"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            elseif data_type == "binary"
+                # Binary data - should be Vector{UInt8}
+                if isa(data, Vector{UInt8})
+                    log_trace("  Type: Vector{UInt8} (binary)")
+                    log_trace("  Size: $(length(data)) bytes")
+                    
+                    # Save to file
+                    output_path = "./received_$dataname"
+                    write(output_path, data)
+                    log_trace("  Saved to: $output_path")
+                else
+                    log_trace("  ERROR: Expected Vector{UInt8}, got $(typeof(data))")
+                end
+                
+            else
+                log_trace("  ERROR: Unknown data type '$data_type'")
+            end
+        end
+        
+        # Summary
+        println("\n=== Verification Summary ===")
+        text_count = count(x -> x[3] == "text", result["payloads"])
+        dict_count = count(x -> x[3] == "dictionary", result["payloads"])
+        arrowtable_count = count(x -> x[3] == "arrowtable", result["payloads"])
+        jsontable_count = count(x -> x[3] == "jsontable", result["payloads"])
+        table_count = count(x -> x[3] == "table", result["payloads"])  # backward compatibility
+        image_count = count(x -> x[3] == "image", result["payloads"])
+        audio_count = count(x -> x[3] == "audio", result["payloads"])
+        video_count = count(x -> x[3] == "video", result["payloads"])
+        binary_count = count(x -> x[3] == "binary", result["payloads"])
+        
+        log_trace("Text payloads: $text_count")
+        log_trace("Dictionary payloads: $dict_count")
+        log_trace("Arrow table payloads: $arrowtable_count")
+        log_trace("JSON table payloads: $jsontable_count")
+        log_trace("Table payloads (backward compat): $table_count")
+        log_trace("Image payloads: $image_count")
+        log_trace("Audio payloads: $audio_count")
+        log_trace("Video payloads: $video_count")
+        log_trace("Binary payloads: $binary_count")
+        
+        # Print transport type info for each payload if available
+        println("\n=== Payload Details ===")
+        for (dataname, data, data_type) in result["payloads"]
+            if data_type in ["image", "audio", "video", "binary"]
+                log_trace("$dataname: $(length(data)) bytes (binary)")
+            elseif data_type == "arrowtable"
+                # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (Arrow.Table)")
+            elseif data_type == "jsontable"
+                log_trace("$dataname: $(length(data)) rows (Vector{Dict/NamedTuple})")
+            elseif data_type == "table"
+                data = DataFrame(data)
+                # log_trace("$dataname: $(size(data, 1)) rows x $(size(data, 2)) columns (DataFrame)")
+            elseif data_type == "dictionary"
+                log_trace("$dataname: $(length(JSON.json(data))) bytes (Dict)")
+            elseif data_type == "text"
+                log_trace("$dataname: $(length(data)) characters (String)")
+            end
+        end
+    end
+    
+    # Keep listening for 2 minutes
+    sleep(180)  
+    NATS.drain(conn)
+end
+
+
+# Run the test
+println("Starting mixed-content transport test...")
+println("Note: This receiver will wait for messages from the sender.")
+println("Run test_julia_to_julia_mix_sender.jl first to send test data.")
+
+# Run receiver
+println("\ntesting smartreceive for mixed content")
+test_mix_receive()
+
+println("\nTest completed.")

test/test_julia_mix_payloads_sender.jl

@@ -0,0 +1,258 @@
+#!/usr/bin/env julia
+# Test script for mixed-content message testing
+# Tests sending a mix of text, dictionary, arrowtable, jsontable, image, audio, video, and binary data
+# from Julia serviceA to Julia serviceB using msghandler.jl smartsend
+# 
+# This test demonstrates that any combination and any number of mixed content
+# can be sent and received correctly.
+# 
+# Key concept: DataFrames are the main table representation in Julia.
+# The msghandler.jl library handles serialization:
+# - For "arrowtable" type: DataFrame is serialized to Arrow IPC format
+# - For "jsontable" type: DataFrame is converted to Vector{Dict} and then to JSON
+
+using NATS, JSON, UUIDs, Dates, PrettyPrinting, DataFrames, Arrow, HTTP, Base64
+
+# Include the bridge module
+include("../src/msghandler.jl")
+using .msghandler
+
+# Configuration
+const SUBJECT = "/msghandler"
+const NATS_URL = "nats.yiem.cc"
+const FILESERVER_URL = "http://192.168.88.104:8080"
+
+# Create correlation ID for tracing
+correlation_id = string(uuid4())
+
+
+# ------------------------------------------------------------------------------------------------ #
+#                                      test mixed content transfer                                 #
+# ------------------------------------------------------------------------------------------------ #
+
+
+# Helper: Log with correlation ID
+function log_trace(message)
+    timestamp = Dates.now()
+    println("[$timestamp] [Correlation: $correlation_id] $message")
+end
+
+
+# File upload handler for plik server
+function plik_upload_handler(fileserver_url::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
+    # Get upload ID
+    url_getUploadID = "$fileserver_url/upload"
+    headers = ["Content-Type" => "application/json"]
+    body = """{ "OneShot" : true }"""
+    httpResponse = HTTP.request("POST", url_getUploadID, headers, body; body_is_form=false)
+    responseJson = JSON.parse(String(httpResponse.body))
+    uploadid = responseJson["id"]
+    uploadtoken = responseJson["uploadToken"]
+    
+    # Upload file
+    file_multipart = HTTP.Multipart(dataname, IOBuffer(data), "application/octet-stream")
+    url_upload = "$fileserver_url/file/$uploadid"
+    headers = ["X-UploadToken" => uploadtoken]
+    
+    form = HTTP.Form(Dict("file" => file_multipart))
+    httpResponse = HTTP.post(url_upload, headers, form)
+    responseJson = JSON.parse(String(httpResponse.body))
+    
+    fileid = responseJson["id"]
+    url = "$fileserver_url/file/$uploadid/$fileid/$dataname"
+    
+    return Dict("status" => httpResponse.status, "uploadid" => uploadid, "fileid" => fileid, "url" => url)
+end
+
+
+# Helper: Create sample data for each type
+function create_sample_data()
+    # Text data (small - direct transport)
+    text_data = "Hello! This is a test chat message. 🎉\nHow are you doing today? 😊"
+    
+    # Dictionary/JSON data (medium - could be direct or link)
+    dict_data = Dict(
+        "type" => "chat",
+        "sender" => "serviceA",
+        "receiver" => "serviceB",
+        "metadata" => Dict(
+            "timestamp" => string(Dates.now()),
+            "priority" => "high",
+            "tags" => ["urgent", "chat", "test"]
+        ),
+        "content" => Dict(
+            "text" => "This is a JSON-formatted chat message with nested structure.",
+            "format" => "markdown",
+            "mentions" => ["user1", "user2"]
+        )
+    )
+    
+    # Arrow table data (DataFrame - small - direct transport)
+    # Uses Arrow IPC format for efficient binary serialization
+    # msghandler.jl handles serialization: DataFrame -> Arrow IPC
+    arrow_table_small = DataFrame(
+        id = 1:10,
+        name = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"],
+        score = rand(50:100, 10),
+        active = rand([true, false], 10)
+    )
+    
+    # Arrow table data (DataFrame - large - link transport)
+    # ~1.5MB of Arrow data (200,000 rows) - should trigger link transport
+    # msghandler.jl handles serialization: DataFrame -> Arrow IPC
+    arrow_table_large = DataFrame(
+        id = 1:2_000_000,
+        name = ["user_$i" for i in 1:2_000_000],
+        score = rand(50:100, 2_000_000),
+        active = rand([true, false], 2_000_000),
+        timestamp = [string(Dates.now()) for _ in 1:2_000_000]
+    )
+    
+    # Json table data (DataFrame - small - direct transport)
+    # Uses JSON format for human-readable tabular data
+    # msghandler.jl handles serialization: DataFrame -> Vector{Dict} -> JSON
+    json_table_small = DataFrame(
+        id = 1:10,
+        name = ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"],
+        score = rand(50:100, 10),
+        active = rand([true, false], 10)
+    )
+    
+    # Json table data (DataFrame - large - link transport)
+    # ~1.5MB of JSON data (150,000 rows) - should trigger link transport
+    # msghandler.jl handles serialization: DataFrame -> Vector{Dict} -> JSON
+    json_table_large = DataFrame(
+        id = 1:2_000_000,
+        name = ["user_$i" for i in 1:2_000_000],
+        score = rand(50:100, 2_000_000),
+        active = rand([true, false], 2_000_000)
+    )
+    
+    # Audio data (small binary - direct transport)
+    audio_data = UInt8[rand(1:255) for _ in 1:100]
+    
+    # Audio data (large - link transport)
+    # ~1.5MB of audio-like data
+    large_audio_data = UInt8[rand(1:255) for _ in 1:1_500_000]
+    
+    # Video data (small binary - direct transport)
+    video_data = UInt8[rand(1:255) for _ in 1:150]
+    
+    # Video data (large - link transport)
+    # ~1.5MB of video-like data
+    large_video_data = UInt8[rand(1:255) for _ in 1:1_500_000]
+    
+    # Binary data (small - direct transport)
+    binary_data = UInt8[rand(1:255) for _ in 1:200]
+    
+    # Binary data (large - link transport)
+    # ~1.5MB of binary data
+    large_binary_data = UInt8[rand(1:255) for _ in 1:1_500_000]
+    
+    return (
+        text_data,
+        dict_data,
+        arrow_table_small,
+        arrow_table_large,
+        json_table_small,
+        json_table_large,
+        audio_data,
+        large_audio_data,
+        video_data,
+        large_video_data,
+        binary_data,
+        large_binary_data
+    )
+end
+
+
+# Sender: Send mixed content via smartsend
+function test_mix_send()
+    # Create sample data
+    (text_data, dict_data, arrow_table_small, arrow_table_large, json_table_small, json_table_large, audio_data, large_audio_data, video_data, large_video_data, binary_data, large_binary_data) = create_sample_data()
+
+    # Read image files from disk (following test_julia_file_sender.jl pattern)
+    # Small image - should use direct transport
+    file_path_small_image = "./test/small_image.jpg"
+    file_data_small_image = read(file_path_small_image)
+    filename_small_image = basename(file_path_small_image)
+
+    # Large image - should use link transport
+    file_path_large_image = "./test/large_image.png"
+    file_data_large_image = read(file_path_large_image)
+    filename_large_image = basename(file_path_large_image)
+
+    # Create payloads list - mixed content with both small and large data
+    # Small data uses direct transport, large data uses link transport
+    # Key: Pass DataFrame directly and specify type as "arrowtable" or "jsontable"
+    # msghandler.jl handles the serialization internally
+    payloads = [
+        # Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image
+        ("chat_text", text_data, "text"),
+        ("chat_json", dict_data, "dictionary"),
+        # ("arrow_table_small", arrow_table_small, "arrowtable"),
+        ("json_table_small", json_table_small, "jsontable"),
+        (filename_small_image, file_data_small_image, "binary"),
+
+        # Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary
+        # ("arrow_table_large", arrow_table_large, "arrowtable"),
+        ("json_table_large", json_table_large, "jsontable"),
+        (filename_large_image, file_data_large_image, "binary"),
+        ("audio_clip_large", large_audio_data, "audio"),
+        ("video_clip_large", large_video_data, "video"),
+        ("binary_file_large", large_binary_data, "binary")
+    ]
+    
+    # Use smartsend with mixed content
+    sendinfo = msghandler.smartsend(
+        SUBJECT,
+        payloads;  # List of (dataname, data, type) tuples
+        broker_url = NATS_URL,
+        fileserver_url = FILESERVER_URL,
+        fileserver_upload_handler = plik_upload_handler,
+        size_threshold = 1_000_000,  # 1MB threshold
+        correlation_id = correlation_id,
+        msg_purpose = "chat",
+        sender_name = "mix_sender",
+        receiver_name = "",
+        receiver_id = "",
+        reply_to = "",
+        reply_to_msg_id = "",
+        is_publish = true  # Publish the message to NATS
+    )
+
+    env, env_json_str = sendinfo
+    log_trace("Sent message with $(length(env.payloads)) payloads")
+    
+    # Log transport type for each payload
+    for (i, payload) in enumerate(env.payloads)
+        log_trace("Payload $i ('$payload.dataname'):")
+        log_trace("  Transport: $(payload.transport)")
+        log_trace("  Type: $(payload.payload_type)")
+        log_trace("  Size: $(payload.size) bytes")
+        log_trace("  Encoding: $(payload.encoding)")
+        
+        if payload.transport == "link"
+            log_trace("  URL: $(payload.data)")
+        end
+    end
+    
+    # Summary
+    println("\n--- Transport Summary ---")
+    direct_count = count(p -> p.transport == "direct", env.payloads)
+    link_count = count(p -> p.transport == "link", env.payloads)
+    log_trace("Direct transport: $direct_count payloads")
+    log_trace("Link transport: $link_count payloads")
+end
+
+
+# Run the test
+println("Starting mixed-content transport test...")
+println("Correlation ID: $correlation_id")
+
+# Run sender
+println("start smartsend for mixed content")
+test_mix_send()
+
+println("\nTest completed.")
+println("Note: Run test_julia_to_julia_mix_receiver.jl to receive the messages.")

test/test_py_mix_payloads_sender.py

@@ -0,0 +1,199 @@
+"""
+Python Mix Payloads Sender Test
+Tests the smartsend function with mixed payload types
+"""
+
+import asyncio
+import sys
+import os
+import base64
+
+# Add parent directory to path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from msghandler import smartsend, DEFAULT_BROKER_URL, DEFAULT_FILESERVER_URL
+
+TEST_SUBJECT = '/test/mix'
+TEST_BROKER_URL = os.environ.get('NATS_URL', 'nats://localhost:4222')
+TEST_FILESERVER_URL = os.environ.get('FILESERVER_URL', 'http://localhost:8080')
+
+
+async def run_test():
+    print('=== Python Mix Payloads Sender Test ===\n')
+    
+    correlation_id = 'py-mix-test-' + str(asyncio.get_event_loop().time() * 1000000)
+    print(f'Correlation ID: {correlation_id}')
+    print(f'Subject: {TEST_SUBJECT}')
+    print(f'Broker URL: {TEST_BROKER_URL}\n')
+    
+    # Test data - mixed payload types
+    text_data = 'Hello, msghandler!'
+    dict_data = {'key1': 'value1', 'key2': 42, 'nested': {'a': 1, 'b': 2}}
+    image_data = bytes([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A])  # PNG header
+    
+    # Table data
+    try:
+        import pandas as pd
+        table_data = pd.DataFrame({
+            'id': [1, 2, 3],
+            'name': ['Alice', 'Bob', 'Charlie'],
+            'age': [30, 25, 35]
+        })
+        table_available = True
+    except ImportError:
+        table_available = False
+        table_data = None
+    
+    test_data = [
+        ('message', text_data, 'text'),
+        ('config', dict_data, 'dictionary'),
+        ('image', image_data, 'image')
+    ]
+    
+    if table_available:
+        test_data.append(('users', table_data, 'table'))
+    
+    try:
+        # Send the message
+        print('Sending mixed payloads...')
+        env, env_json_str = await smartsend(
+            TEST_SUBJECT,
+            test_data,
+            broker_url=TEST_BROKER_URL,
+            fileserver_url=TEST_FILESERVER_URL,
+            correlation_id=correlation_id,
+            msg_purpose='test',
+            sender_name='py-mix-test',
+            is_publish=False
+        )
+        
+        print('\n=== Envelope Created ===')
+        print(f'Correlation ID: {env["correlation_id"]}')
+        print(f'Message ID: {env["msg_id"]}')
+        print(f'Timestamp: {env["timestamp"]}')
+        print(f'Subject: {env["send_to"]}')
+        print(f'Purpose: {env["msg_purpose"]}')
+        print(f'Sender: {env["sender_name"]}')
+        print(f'Payloads: {len(env["payloads"])}\n')
+        
+        # Validate envelope structure
+        print('=== Validation ===')
+        passed = True
+        
+        expected_count = 4 if table_available else 3
+        if len(env['payloads']) != expected_count:
+            print(f'❌ Expected {expected_count} payloads, got {len(env["payloads"])}')
+            passed = False
+        else:
+            print('✅ Correct number of payloads')
+        
+        # Test each payload
+        expected_datanames = ['message', 'config', 'image']
+        expected_types = ['text', 'dictionary', 'image']
+        expected_data = [text_data, dict_data, image_data]
+        
+        if table_available:
+            expected_datanames.append('users')
+            expected_types.append('table')
+        
+        for i in range(len(env['payloads'])):
+            payload = env['payloads'][i]
+            
+            if payload['dataname'] != expected_datanames[i]:
+                print(f"❌ Payload {i + 1}: Expected dataname '{expected_datanames[i]}', got '{payload['dataname']}'")
+                passed = False
+            else:
+                print(f'✅ Payload {i + 1}: Correct dataname')
+            
+            if payload['payload_type'] != expected_types[i]:
+                print(f"❌ Payload {i + 1}: Expected type '{expected_types[i]}', got '{payload['payload_type']}'")
+                passed = False
+            else:
+                print(f'✅ Payload {i + 1}: Correct type')
+            
+            if payload['transport'] != 'direct':
+                print(f"❌ Payload {i + 1}: Expected transport 'direct', got '{payload['transport']}'")
+                passed = False
+            else:
+                print(f'✅ Payload {i + 1}: Correct transport')
+            
+            if payload['encoding'] != 'base64':
+                print(f"❌ Payload {i + 1}: Expected encoding 'base64', got '{payload['encoding']}'")
+                passed = False
+            else:
+                print(f'✅ Payload {i + 1}: Correct encoding')
+            
+            # Verify data integrity based on type
+            decoded_data = base64.b64decode(payload['data'])
+            
+            if expected_types[i] == 'text':
+                decoded_text = decoded_data.decode('utf8')
+                if decoded_text != expected_data[i]:
+                    print(f'❌ Payload {i + 1}: Data integrity mismatch')
+                    passed = False
+                else:
+                    print(f'✅ Payload {i + 1}: Data integrity verified')
+            elif expected_types[i] == 'dictionary':
+                import json
+                decoded_dict = json.loads(decoded_data.decode('utf8'))
+                if json.dumps(decoded_dict, sort_keys=True) != json.dumps(expected_data[i], sort_keys=True):
+                    print(f'❌ Payload {i + 1}: Data integrity mismatch')
+                    passed = False
+                else:
+                    print(f'✅ Payload {i + 1}: Data integrity verified')
+            elif expected_types[i] == 'image':
+                if decoded_data != expected_data[i]:
+                    print(f'❌ Payload {i + 1}: Data integrity mismatch')
+                    passed = False
+                else:
+                    print(f'✅ Payload {i + 1}: Data integrity verified')
+            elif expected_types[i] == 'table':
+                if len(decoded_data) > 0:
+                    print(f'✅ Payload {i + 1}: Arrow IPC data present ({len(decoded_data)} bytes)')
+                else:
+                    print(f'❌ Payload {i + 1}: Arrow IPC data is empty')
+                    passed = False
+            
+            print(f'   Size: {payload["size"]} bytes\n')
+        
+        # Test with chat-like payload (text + image + audio)
+        print('=== Chat-like Payload Test ===')
+        chat_data = [
+            ('text', 'Hello!', 'text'),
+            ('image', bytes([0xFF, 0xD8, 0xFF, 0xE0]), 'image'),
+            ('audio', bytes([0x46, 0x4C, 0x41, 0x43]), 'audio')
+        ]
+        
+        chat_env, _ = await smartsend(
+            TEST_SUBJECT,
+            chat_data,
+            broker_url=TEST_BROKER_URL,
+            fileserver_url=TEST_FILESERVER_URL,
+            correlation_id='chat-' + correlation_id,
+            is_publish=False
+        )
+        
+        if len(chat_env['payloads']) == 3:
+            print('✅ Chat-like payloads handled correctly')
+        else:
+            print('❌ Chat-like payloads handling failed')
+            passed = False
+        
+        # Final result
+        print('\n=== Test Result ===')
+        if passed:
+            print('✅ ALL TESTS PASSED')
+            sys.exit(0)
+        else:
+            print('❌ SOME TESTS FAILED')
+            sys.exit(1)
+            
+    except Exception as e:
+        print(f'❌ Test failed with error: {e}')
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    asyncio.run(run_test())