update
This commit is contained in:
@@ -151,14 +151,14 @@ async function runTest() {
|
|||||||
// Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image
|
// Small data (direct transport) - text, dictionary, arrowtable, jsontable, small image
|
||||||
['chat_text', textData, 'text'],
|
['chat_text', textData, 'text'],
|
||||||
['chat_json', dictData, 'dictionary'],
|
['chat_json', dictData, 'dictionary'],
|
||||||
['arrow_table_small', arrowTableSmall, 'arrowtable'],
|
// ['arrow_table_small', arrowTableSmall, 'arrowtable'],
|
||||||
['json_table_small', jsonTableSmall, 'jsontable'],
|
['json_table_small', jsonTableSmall, 'jsontable'],
|
||||||
[filename_small_image, file_data_small_image, 'binary'],
|
[filename_small_image, file_data_small_image, 'binary'],
|
||||||
|
|
||||||
// Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary
|
// Large data (link transport) - large arrowtable, large jsontable, large image, large audio, large video, large binary
|
||||||
// ['arrow_table_large', largeArrowTable, 'arrowtable'],
|
// ['arrow_table_large', largeArrowTable, 'arrowtable'],
|
||||||
['json_table_large', largeJsonTable, 'jsontable'],
|
['json_table_large', largeJsonTable, 'jsontable'],
|
||||||
// [filename_large_image, file_data_large_image, 'binary'],
|
[filename_large_image, file_data_large_image, 'binary'],
|
||||||
// ['audio_clip_large', largeAudioData, 'audio'],
|
// ['audio_clip_large', largeAudioData, 'audio'],
|
||||||
// ['video_clip_large', largeVideoData, 'video'],
|
// ['video_clip_large', largeVideoData, 'video'],
|
||||||
// ['binary_file_large', largeBinaryData, 'binary']
|
// ['binary_file_large', largeBinaryData, 'binary']
|
||||||
@@ -197,176 +197,6 @@ async function runTest() {
|
|||||||
console.log(`Sender: ${env.sender_name}`);
|
console.log(`Sender: ${env.sender_name}`);
|
||||||
console.log(`Payloads: ${env.payloads.length}\n`);
|
console.log(`Payloads: ${env.payloads.length}\n`);
|
||||||
|
|
||||||
// Log transport type for each payload
|
|
||||||
console.log('=== Payload Details ===');
|
|
||||||
for (let i = 0; i < env.payloads.length; i++) {
|
|
||||||
const payload = env.payloads[i];
|
|
||||||
logTrace(`Payload ${i + 1} ('${payload.dataname}'):`);
|
|
||||||
logTrace(` Transport: ${payload.transport}`);
|
|
||||||
logTrace(` Type: ${payload.payload_type}`);
|
|
||||||
logTrace(` Size: ${payload.size} bytes`);
|
|
||||||
logTrace(` Encoding: ${payload.encoding}`);
|
|
||||||
|
|
||||||
if (payload.transport === 'link') {
|
|
||||||
logTrace(` URL: ${payload.data}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Summary
|
|
||||||
console.log('\n=== Transport Summary ===');
|
|
||||||
const directCount = env.payloads.filter(p => p.transport === 'direct').length;
|
|
||||||
const linkCount = env.payloads.filter(p => p.transport === 'link').length;
|
|
||||||
logTrace(`Direct transport: ${directCount} payloads`);
|
|
||||||
logTrace(`Link transport: ${linkCount} payloads`);
|
|
||||||
|
|
||||||
// Validate envelope structure
|
|
||||||
console.log('\n=== Validation ===');
|
|
||||||
let passed = true;
|
|
||||||
|
|
||||||
if (env.payloads.length !== 11) {
|
|
||||||
console.log(`❌ Expected 11 payloads, got ${env.payloads.length}`);
|
|
||||||
passed = false;
|
|
||||||
} else {
|
|
||||||
console.log('✅ Correct number of payloads');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test each payload
|
|
||||||
const expectedDatanames = [
|
|
||||||
'chat_text', 'chat_json', 'arrow_table_small', 'json_table_small',
|
|
||||||
filename_small_image, 'arrow_table_large', 'json_table_large',
|
|
||||||
filename_large_image, 'audio_clip_large', 'video_clip_large',
|
|
||||||
'binary_file_large'
|
|
||||||
];
|
|
||||||
const expectedTypes = [
|
|
||||||
'text', 'dictionary', 'arrowtable', 'jsontable', 'binary',
|
|
||||||
'arrowtable', 'jsontable', 'binary', 'audio', 'video', 'binary'
|
|
||||||
];
|
|
||||||
|
|
||||||
for (let i = 0; i < env.payloads.length; i++) {
|
|
||||||
const payload = env.payloads[i];
|
|
||||||
|
|
||||||
if (payload.dataname !== expectedDatanames[i]) {
|
|
||||||
console.log(`❌ Payload ${i + 1}: Expected dataname '${expectedDatanames[i]}', got '${payload.dataname}'`);
|
|
||||||
passed = false;
|
|
||||||
} else {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Correct dataname ('${payload.dataname}')`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (payload.payload_type !== expectedTypes[i]) {
|
|
||||||
console.log(`❌ Payload ${i + 1}: Expected type '${expectedTypes[i]}', got '${payload.payload_type}'`);
|
|
||||||
passed = false;
|
|
||||||
} else {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Correct type ('${payload.payload_type}')`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate transport based on expected size
|
|
||||||
if (i < 5 || i >= 6 && i <= 10) {
|
|
||||||
// First 5 should be direct (small), rest should be link (large)
|
|
||||||
const shouldBeDirect = i < 5;
|
|
||||||
const isDirect = payload.transport === 'direct';
|
|
||||||
|
|
||||||
if (shouldBeDirect && isDirect) {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Correct transport (direct)`);
|
|
||||||
} else if (!shouldBeDirect && payload.transport === 'link') {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Correct transport (link)`);
|
|
||||||
} else {
|
|
||||||
console.log(`❌ Payload ${i + 1}: Expected ${shouldBeDirect ? 'direct' : 'link'} transport, got '${payload.transport}'`);
|
|
||||||
passed = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate encoding based on payload type
|
|
||||||
let expectedEncoding;
|
|
||||||
if (payload.payload_type === 'jsontable') {
|
|
||||||
expectedEncoding = 'json';
|
|
||||||
} else if (payload.payload_type === 'arrowtable') {
|
|
||||||
expectedEncoding = 'arrow-ipc';
|
|
||||||
} else {
|
|
||||||
expectedEncoding = 'base64';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (payload.encoding !== expectedEncoding) {
|
|
||||||
console.log(`❌ Payload ${i + 1}: Expected encoding '${expectedEncoding}', got '${payload.encoding}'`);
|
|
||||||
passed = false;
|
|
||||||
} else {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Correct encoding ('${payload.encoding}')`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate size field
|
|
||||||
if (payload.size > 0) {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Size field present (${payload.size} bytes)`);
|
|
||||||
} else {
|
|
||||||
console.log(`❌ Payload ${i + 1}: Size field is 0`);
|
|
||||||
passed = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate ID field
|
|
||||||
if (payload.id && payload.id.length > 0) {
|
|
||||||
console.log(`✅ Payload ${i + 1}: ID field present`);
|
|
||||||
} else {
|
|
||||||
console.log(`❌ Payload ${i + 1}: ID field is empty`);
|
|
||||||
passed = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate metadata field
|
|
||||||
if (payload.metadata !== undefined && payload.metadata !== null) {
|
|
||||||
console.log(`✅ Payload ${i + 1}: Metadata field present`);
|
|
||||||
} else {
|
|
||||||
console.log(`❌ Payload ${i + 1}: Metadata field is missing`);
|
|
||||||
passed = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test with chat-like payload (text + image + audio)
|
|
||||||
console.log('=== Chat-like Payload Test ===');
|
|
||||||
const chatData = [
|
|
||||||
['text', 'Hello!', 'text'],
|
|
||||||
['image', Buffer.from([0xFF, 0xD8, 0xFF, 0xE0]), 'image'],
|
|
||||||
['audio', Buffer.from([0x46, 0x4C, 0x41, 0x43]), 'audio']
|
|
||||||
];
|
|
||||||
|
|
||||||
const [chatEnv, _] = await NATSBridge.smartsend(
|
|
||||||
TEST_SUBJECT,
|
|
||||||
chatData,
|
|
||||||
{
|
|
||||||
broker_url: TEST_BROKER_URL,
|
|
||||||
fileserver_url: TEST_FILESERVER_URL,
|
|
||||||
correlation_id: 'chat-' + correlationId,
|
|
||||||
msg_purpose: 'chat',
|
|
||||||
sender_name: 'js-mix-test',
|
|
||||||
is_publish: true
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
if (chatEnv.payloads.length === 3) {
|
|
||||||
console.log('✅ Chat-like payloads handled correctly (3 payloads)');
|
|
||||||
} else {
|
|
||||||
console.log(`❌ Chat-like payloads handling failed (expected 3, got ${chatEnv.payloads.length})`);
|
|
||||||
passed = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify all payload types in chat are direct transport
|
|
||||||
const allDirect = chatEnv.payloads.every(p => p.transport === 'direct');
|
|
||||||
if (allDirect) {
|
|
||||||
console.log('✅ All chat payloads use direct transport (small size)');
|
|
||||||
} else {
|
|
||||||
console.log('❌ Some chat payloads use link transport (unexpected for small data)');
|
|
||||||
passed = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Final result
|
|
||||||
console.log('\n=== Test Result ===');
|
|
||||||
if (passed) {
|
|
||||||
console.log('✅ ALL TESTS PASSED');
|
|
||||||
console.log('\nNote: Run test_js_mix_payloads_receiver.js to receive the messages.');
|
|
||||||
process.exit(0);
|
|
||||||
} else {
|
|
||||||
console.log('❌ SOME TESTS FAILED');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('\n❌ Test failed with error:', error.message);
|
console.error('\n❌ Test failed with error:', error.message);
|
||||||
console.error(error.stack);
|
console.error(error.stack);
|
||||||
|
|||||||
@@ -126,13 +126,6 @@ function test_mix_receive()
|
|||||||
json_str = JSON.json(data, 2)
|
json_str = JSON.json(data, 2)
|
||||||
write(output_path, json_str)
|
write(output_path, json_str)
|
||||||
log_trace(" Saved to: $output_path")
|
log_trace(" Saved to: $output_path")
|
||||||
|
|
||||||
# Also save as Arrow file
|
|
||||||
output_path_arrow = "./received_$dataname.arrow"
|
|
||||||
io = IOBuffer()
|
|
||||||
Arrow.write(io, df)
|
|
||||||
write(output_path_arrow, take!(io))
|
|
||||||
log_trace(" Saved to: $output_path_arrow")
|
|
||||||
else
|
else
|
||||||
log_trace(" ERROR: Expected Vector{Dict/NamedTuple}, got $(typeof(data))")
|
log_trace(" ERROR: Expected Vector{Dict/NamedTuple}, got $(typeof(data))")
|
||||||
end
|
end
|
||||||
@@ -241,7 +234,7 @@ function test_mix_receive()
|
|||||||
end
|
end
|
||||||
|
|
||||||
# Keep listening for 2 minutes
|
# Keep listening for 2 minutes
|
||||||
sleep(120)
|
sleep(180)
|
||||||
NATS.drain(conn)
|
NATS.drain(conn)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@@ -101,11 +101,11 @@ function create_sample_data()
|
|||||||
# ~1.5MB of Arrow data (200,000 rows) - should trigger link transport
|
# ~1.5MB of Arrow data (200,000 rows) - should trigger link transport
|
||||||
# NATSBridge.jl handles serialization: DataFrame -> Arrow IPC
|
# NATSBridge.jl handles serialization: DataFrame -> Arrow IPC
|
||||||
arrow_table_large = DataFrame(
|
arrow_table_large = DataFrame(
|
||||||
id = 1:200_000,
|
id = 1:2_000_000,
|
||||||
name = ["user_$i" for i in 1:200_000],
|
name = ["user_$i" for i in 1:2_000_000],
|
||||||
score = rand(50:100, 200_000),
|
score = rand(50:100, 2_000_000),
|
||||||
active = rand([true, false], 200_000),
|
active = rand([true, false], 2_000_000),
|
||||||
timestamp = [string(Dates.now()) for _ in 1:200_000]
|
timestamp = [string(Dates.now()) for _ in 1:2_000_000]
|
||||||
)
|
)
|
||||||
|
|
||||||
# Json table data (DataFrame - small - direct transport)
|
# Json table data (DataFrame - small - direct transport)
|
||||||
@@ -122,10 +122,10 @@ function create_sample_data()
|
|||||||
# ~1.5MB of JSON data (150,000 rows) - should trigger link transport
|
# ~1.5MB of JSON data (150,000 rows) - should trigger link transport
|
||||||
# NATSBridge.jl handles serialization: DataFrame -> Vector{Dict} -> JSON
|
# NATSBridge.jl handles serialization: DataFrame -> Vector{Dict} -> JSON
|
||||||
json_table_large = DataFrame(
|
json_table_large = DataFrame(
|
||||||
id = 1:1_500_000,
|
id = 1:2_000_000,
|
||||||
name = ["user_$i" for i in 1:1_500_000],
|
name = ["user_$i" for i in 1:2_000_000],
|
||||||
score = rand(50:100, 1_500_000),
|
score = rand(50:100, 2_000_000),
|
||||||
active = rand([true, false], 1_500_000)
|
active = rand([true, false], 2_000_000)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Audio data (small binary - direct transport)
|
# Audio data (small binary - direct transport)
|
||||||
|
|||||||
Reference in New Issue
Block a user