/** * NATSBridge.js - Bi-Directional Data Bridge for JavaScript * Implements smartsend and smartreceive for NATS communication * * This module provides functionality for sending and receiving data across network boundaries * using NATS as the message bus, with support for both direct payload transport and * URL-based transport for larger payloads. * * File Server Handler Architecture: * The system uses handler functions to abstract file server operations, allowing support * for different file server implementations (e.g., Plik, AWS S3, custom HTTP server). * * Handler Function Signatures: * * ```javascript * // Upload handler - uploads data to file server and returns URL * // The handler is passed to smartsend as fileserverUploadHandler parameter * // It receives: (fileserver_url, dataname, data) * // Returns: { status, uploadid, fileid, url } * async function plik_oneshot_upload(fileserver_url, dataname, data) { ... } * * // Download handler - fetches data from file server URL with exponential backoff * // The handler is passed to smartreceive as fileserverDownloadHandler parameter * // It receives: (url, max_retries, base_delay, max_delay, correlation_id) * // Returns: ArrayBuffer (the downloaded data) * async function fileserverDownloadHandler(url, max_retries, base_delay, max_delay, correlation_id) { ... } * ``` * * Multi-Payload Support (Standard API): * The system uses a standardized list-of-tuples format for all payload operations. * Even when sending a single payload, the user must wrap it in a list. * * API Standard: * ```javascript * // Input format for smartsend (always a list of tuples with type info) * [{ dataname, data, type }, ...] * * // Output format for smartreceive (always returns a list of tuples) * [{ dataname, data, type }, ...] * ``` * * Supported types: "text", "dictionary", "table", "image", "audio", "video", "binary" */ // ---------------------------------------------- 100 --------------------------------------------- # // Constants const DEFAULT_SIZE_THRESHOLD = 1_000_000; // 1MB - threshold for switching from direct to link transport const DEFAULT_NATS_URL = "nats://localhost:4222"; // Default NATS server URL const DEFAULT_FILESERVER_URL = "http://localhost:8080"; // Default HTTP file server URL for link transport // Helper: Generate UUID v4 function uuid4() { // Simple UUID v4 generator return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); return v.toString(16); }); } // Helper: Log with correlation ID and timestamp function log_trace(correlation_id, message) { const timestamp = new Date().toISOString(); console.log(`[${timestamp}] [Correlation: ${correlation_id}] ${message}`); } // Helper: Get size of data in bytes function getDataSize(data) { if (typeof data === 'string') { return new TextEncoder().encode(data).length; } else if (data instanceof ArrayBuffer || data instanceof Uint8Array) { return data.byteLength; } else if (typeof data === 'object' && data !== null) { // For objects, serialize to JSON and measure return new TextEncoder().encode(JSON.stringify(data)).length; } return 0; } // Helper: Convert ArrayBuffer to Base64 string function arrayBufferToBase64(buffer) { const bytes = new Uint8Array(buffer); let binary = ''; for (let i = 0; i < bytes.length; i++) { binary += String.fromCharCode(bytes[i]); } return btoa(binary); } // Helper: Convert Base64 string to ArrayBuffer function base64ToArrayBuffer(base64) { const binaryString = atob(base64); const len = binaryString.length; const bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes.buffer; } // Helper: Convert Uint8Array to Base64 string function uint8ArrayToBase64(uint8array) { let binary = ''; for (let i = 0; i < uint8array.byteLength; i++) { binary += String.fromCharCode(uint8array[i]); } return btoa(binary); } // Helper: Convert Base64 string to Uint8Array function base64ToUint8Array(base64) { const binaryString = atob(base64); const len = binaryString.length; const bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes; } // Helper: Serialize data based on type function _serialize_data(data, type) { /** * Serialize data according to specified format * * Supported formats: * - "text": Treats data as text and converts to UTF-8 bytes * - "dictionary": Serializes data as JSON and returns the UTF-8 byte representation * - "table": Serializes data as an Arrow IPC stream (table format) - NOT IMPLEMENTED (requires arrow library) * - "image": Expects binary data (ArrayBuffer) and returns it as bytes * - "audio": Expects binary data (ArrayBuffer) and returns it as bytes * - "video": Expects binary data (ArrayBuffer) and returns it as bytes * - "binary": Generic binary data (ArrayBuffer or Uint8Array) and returns bytes */ if (type === "text") { if (typeof data === 'string') { return new TextEncoder().encode(data); } else { throw new Error("Text data must be a String"); } } else if (type === "dictionary") { // JSON data - serialize directly const jsonStr = JSON.stringify(data); return new TextEncoder().encode(jsonStr); } else if (type === "table") { // Table data - convert to Arrow IPC stream (NOT IMPLEMENTED in pure JavaScript) // This would require the apache-arrow library throw new Error("Table serialization requires apache-arrow library"); } else if (type === "image") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Image data must be ArrayBuffer or Uint8Array"); } } else if (type === "audio") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Audio data must be ArrayBuffer or Uint8Array"); } } else if (type === "video") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Video data must be ArrayBuffer or Uint8Array"); } } else if (type === "binary") { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else { throw new Error("Binary data must be ArrayBuffer or Uint8Array"); } } else { throw new Error(`Unknown type: ${type}`); } } // Helper: Deserialize bytes based on type function _deserialize_data(data, type, correlation_id) { /** * Deserialize bytes to data based on type * * Supported formats: * - "text": Converts bytes to string * - "dictionary": Parses JSON string * - "table": Parses Arrow IPC stream - NOT IMPLEMENTED (requires apache-arrow library) * - "image": Returns binary data * - "audio": Returns binary data * - "video": Returns binary data * - "binary": Returns binary data */ if (type === "text") { const decoder = new TextDecoder(); return decoder.decode(data); } else if (type === "dictionary") { const decoder = new TextDecoder(); const jsonStr = decoder.decode(data); return JSON.parse(jsonStr); } else if (type === "table") { // Table data - deserialize Arrow IPC stream (NOT IMPLEMENTED in pure JavaScript) throw new Error("Table deserialization requires apache-arrow library"); } else if (type === "image") { return data; } else if (type === "audio") { return data; } else if (type === "video") { return data; } else if (type === "binary") { return data; } else { throw new Error(`Unknown type: ${type}`); } } // Helper: Upload data to file server // Internal wrapper that adds correlation_id logging for smartsend async function _upload_to_fileserver(fileserver_url, dataname, data, correlation_id) { /** * Internal upload helper - wraps plik_oneshot_upload to add correlation_id logging * This allows smartsend to pass correlation_id for tracing without changing the handler signature */ log_trace(correlation_id, `Uploading ${dataname} to fileserver: ${fileserver_url}`); const result = await plik_oneshot_upload(fileserver_url, dataname, data); log_trace(correlation_id, `Uploaded to URL: ${result.url}`); return result; } // Helper: Fetch data from URL with exponential backoff async function _fetch_with_backoff(url, max_retries, base_delay, max_delay, correlation_id) { /** * Fetch data from URL with retry logic using exponential backoff */ let delay = base_delay; for (let attempt = 1; attempt <= max_retries; attempt++) { try { const response = await fetch(url); if (response.status === 200) { log_trace(correlation_id, `Successfully fetched data from ${url} on attempt ${attempt}`); const arrayBuffer = await response.arrayBuffer(); return new Uint8Array(arrayBuffer); } else { throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`); } } catch (e) { log_trace(correlation_id, `Attempt ${attempt} failed: ${e.message}`); if (attempt < max_retries) { // Sleep with exponential backoff await new Promise(resolve => setTimeout(resolve, delay)); delay = Math.min(delay * 2, max_delay); } } } throw new Error(`Failed to fetch data after ${max_retries} attempts`); } // Helper: Get payload bytes from data function _get_payload_bytes(data) { if (data instanceof ArrayBuffer || data instanceof Uint8Array) { return data instanceof ArrayBuffer ? new Uint8Array(data) : data; } else if (typeof data === 'string') { return new TextEncoder().encode(data); } else { // For objects, serialize to JSON return new TextEncoder().encode(JSON.stringify(data)); } } // MessagePayload class - matches msg_payload_v1 Julia struct class MessagePayload { /** * Represents a single payload in the message envelope * Matches Julia's msg_payload_v1 struct * * @param {Object} options - Payload options * @param {string} options.id - ID of this payload (e.g., "uuid4") * @param {string} options.dataname - Name of this payload (e.g., "login_image") * @param {string} options.payload_type - Payload type: "text", "dictionary", "table", "image", "audio", "video", "binary" * @param {string} options.transport - "direct" or "link" * @param {string} options.encoding - "none", "json", "base64", "arrow-ipc" * @param {number} options.size - Data size in bytes * @param {string|Uint8Array} options.data - Payload data (Uint8Array for direct, URL string for link) * @param {Object} options.metadata - Metadata for this payload */ constructor(options) { this.id = options.id || uuid4(); this.dataname = options.dataname; this.payload_type = options.payload_type; this.transport = options.transport; this.encoding = options.encoding; this.size = options.size; this.data = options.data; this.metadata = options.metadata || {}; } // Convert to JSON object - uses snake_case to match Julia API toJSON() { const obj = { id: this.id, dataname: this.dataname, payload_type: this.payload_type, transport: this.transport, encoding: this.encoding, size: this.size }; // Include data based on transport type if (this.transport === "direct" && this.data !== null && this.data !== undefined) { if (this.encoding === "base64" || this.encoding === "json") { obj.data = this.data; } else { // For other encodings, use base64 const payloadBytes = _get_payload_bytes(this.data); obj.data = uint8ArrayToBase64(payloadBytes); } } else if (this.transport === "link" && this.data !== null && this.data !== undefined) { // For link transport, data is a URL string obj.data = this.data; } if (Object.keys(this.metadata).length > 0) { obj.metadata = this.metadata; } return obj; } } // MessageEnvelope class - matches msg_envelope_v1 Julia struct class MessageEnvelope { /** * Represents the message envelope containing metadata and payloads * Matches Julia's msg_envelope_v1 struct * * @param {Object} options - Envelope options * @param {string} options.correlation_id - Unique identifier to track messages * @param {string} options.msg_id - This message id * @param {string} options.timestamp - Message published timestamp * @param {string} options.send_to - Topic/subject the sender sends to * @param {string} options.msg_purpose - Purpose of this message * @param {string} options.sender_name - Name of the sender * @param {string} options.sender_id - UUID of the sender * @param {string} options.receiver_name - Name of the receiver * @param {string} options.receiver_id - UUID of the receiver * @param {string} options.reply_to - Topic to reply to * @param {string} options.reply_to_msg_id - Message id this message is replying to * @param {string} options.broker_url - NATS server address * @param {Object} options.metadata - Metadata for the envelope * @param {Array} options.payloads - Array of payloads */ constructor(options) { this.correlation_id = options.correlation_id || uuid4(); this.msg_id = options.msg_id || uuid4(); this.timestamp = options.timestamp || new Date().toISOString(); this.send_to = options.send_to; this.msg_purpose = options.msg_purpose || ""; this.sender_name = options.sender_name || ""; this.sender_id = options.sender_id || uuid4(); this.receiver_name = options.receiver_name || ""; this.receiver_id = options.receiver_id || ""; this.reply_to = options.reply_to || ""; this.reply_to_msg_id = options.reply_to_msg_id || ""; this.broker_url = options.broker_url || DEFAULT_NATS_URL; this.metadata = options.metadata || {}; this.payloads = options.payloads || []; } // Convert to JSON object - uses snake_case to match Julia API toJSON() { const obj = { correlation_id: this.correlation_id, msg_id: this.msg_id, timestamp: this.timestamp, send_to: this.send_to, msg_purpose: this.msg_purpose, sender_name: this.sender_name, sender_id: this.sender_id, receiver_name: this.receiver_name, receiver_id: this.receiver_id, reply_to: this.reply_to, reply_to_msg_id: this.reply_to_msg_id, broker_url: this.broker_url }; if (Object.keys(this.metadata).length > 0) { obj.metadata = this.metadata; } if (this.payloads.length > 0) { obj.payloads = this.payloads.map(p => p.toJSON()); } return obj; } // Convert to JSON string toString() { return JSON.stringify(this.toJSON()); } } // SmartSend function - matches Julia smartsend signature and behavior async function smartsend(subject, data, options = {}) { /** * Send data either directly via NATS or via a fileserver URL, depending on payload size * * This function intelligently routes data delivery based on payload size relative to a threshold. * If the serialized payload is smaller than `size_threshold`, it encodes the data as Base64 and publishes directly over NATS. * Otherwise, it uploads the data to a fileserver and publishes only the download URL over NATS. * * @param {string} subject - NATS subject to publish the message to * @param {Array} data - List of {dataname, data, type} objects to send (must be a list, even for single payload) * @param {Object} options - Additional options * @param {string} options.broker_url - URL of the NATS server (default: "nats://localhost:4222") * @param {string} options.fileserver_url - Base URL of the file server (default: "http://localhost:8080") * @param {Function} options.fileserver_upload_handler - Function to handle fileserver uploads * @param {number} options.size_threshold - Threshold in bytes separating direct vs link transport (default: 1MB) * @param {string} options.correlation_id - Optional correlation ID for tracing * @param {string} options.msg_purpose - Purpose of the message (default: "chat") * @param {string} options.sender_name - Name of the sender (default: "NATSBridge") * @param {string} options.receiver_name - Name of the receiver (default: "") * @param {string} options.receiver_id - UUID of the receiver (default: "") * @param {string} options.reply_to - Topic to reply to (default: "") * @param {string} options.reply_to_msg_id - Message ID this message is replying to (default: "") * @param {boolean} options.is_publish - Whether to automatically publish the message to NATS (default: true) * - When true: Message is published to NATS automatically * - When false: Returns (env, env_json_str) without publishing, allowing manual publishing * @returns {Promise} - A tuple-like object with { env: MessageEnvelope, env_json_str: string } * - env: MessageEnvelope object with all metadata and payloads * - env_json_str: JSON string representation of the envelope for manual publishing */ const { broker_url = DEFAULT_NATS_URL, fileserver_url = DEFAULT_FILESERVER_URL, fileserver_upload_handler = _upload_to_fileserver, size_threshold = DEFAULT_SIZE_THRESHOLD, correlation_id = uuid4(), msg_purpose = "chat", sender_name = "NATSBridge", receiver_name = "", receiver_id = "", reply_to = "", reply_to_msg_id = "", is_publish = true // Whether to automatically publish the message to NATS } = options; log_trace(correlation_id, `Starting smartsend for subject: ${subject}`); // Generate message metadata const msg_id = uuid4(); // Process each payload in the list const payloads = []; for (const payload of data) { const dataname = payload.dataname; const payloadData = payload.data; const payloadType = payload.type; // Serialize data based on type const payloadBytes = _serialize_data(payloadData, payloadType); const payloadSize = payloadBytes.byteLength; log_trace(correlation_id, `Serialized payload '${dataname}' (payload_type: ${payloadType}) size: ${payloadSize} bytes`); // Decision: Direct vs Link if (payloadSize < size_threshold) { // Direct path - Base64 encode and send via NATS const payloadB64 = uint8ArrayToBase64(payloadBytes); log_trace(correlation_id, `Using direct transport for ${payloadSize} bytes`); // Create MessagePayload for direct transport const payloadObj = new MessagePayload({ dataname: dataname, payload_type: payloadType, transport: "direct", encoding: "base64", size: payloadSize, data: payloadB64, metadata: { payload_bytes: payloadSize } }); payloads.push(payloadObj); } else { // Link path - Upload to HTTP server, send URL via NATS log_trace(correlation_id, `Using link transport, uploading to fileserver`); // Upload to HTTP server using plik_oneshot_upload handler const response = await fileserver_upload_handler(fileserver_url, dataname, payloadBytes); if (response.status !== 200) { throw new Error(`Failed to upload data to fileserver: ${response.status}`); } const url = response.url; log_trace(correlation_id, `Uploaded to URL: ${url}`); // Create MessagePayload for link transport const payloadObj = new MessagePayload({ dataname: dataname, payload_type: payloadType, transport: "link", encoding: "none", size: payloadSize, data: url, metadata: {} }); payloads.push(payloadObj); } } // Create MessageEnvelope with all payloads const env = new MessageEnvelope({ correlation_id: correlation_id, msg_id: msg_id, send_to: subject, msg_purpose: msg_purpose, sender_name: sender_name, receiver_name: receiver_name, receiver_id: receiver_id, reply_to: reply_to, reply_to_msg_id: reply_to_msg_id, broker_url: broker_url, payloads: payloads }); // Convert envelope to JSON string const env_json_str = env.toString(); // Publish to NATS if isPublish is true if (is_publish) { await publish_message(broker_url, subject, env_json_str, correlation_id); } // Return both envelope and JSON string (tuple-like structure, matching Julia API) return { env: env, env_json_str: env_json_str }; } // Helper: Publish message to NATS async function publish_message(broker_url, subject, message, correlation_id) { /** * Publish a message to a NATS subject with proper connection management * * @param {string} broker_url - NATS server URL * @param {string} subject - NATS subject to publish to * @param {string} message - JSON message to publish * @param {string} correlation_id - Correlation ID for logging */ log_trace(correlation_id, `Publishing message to ${subject}`); // For Node.js, we would use nats.js library // This is a placeholder that throws an error // In production, you would import and use the actual nats library // Example with nats.js: // import { connect } from 'nats'; // const nc = await connect({ servers: [broker_url] }); // await nc.publish(subject, message); // nc.close(); // For now, just log the message console.log(`[NATS PUBLISH] Subject: ${subject}, Message: ${message.substring(0, 100)}...`); } // SmartReceive function - matches Julia smartreceive signature and behavior async function smartreceive(msg, options = {}) { /** * Receive and process messages from NATS * * This function processes incoming NATS messages, handling both direct transport * (base64 decoded payloads) and link transport (URL-based payloads). * * @param {Object} msg - NATS message object with payload property * @param {Object} options - Additional options * @param {Function} options.fileserver_download_handler - Function to handle downloading data from file server URLs * @param {number} options.max_retries - Maximum retry attempts for fetching URL (default: 5) * @param {number} options.base_delay - Initial delay for exponential backoff in ms (default: 100) * @param {number} options.max_delay - Maximum delay for exponential backoff in ms (default: 5000) * * @returns {Promise} - JSON object of envelope with payloads field containing list of {dataname, data, type} tuples */ const { fileserver_download_handler = _fetch_with_backoff, max_retries = 5, base_delay = 100, max_delay = 5000 } = options; // Parse the JSON envelope const jsonStr = typeof msg.payload === 'string' ? msg.payload : new TextDecoder().decode(msg.payload); const json_data = JSON.parse(jsonStr); log_trace(json_data.correlation_id, `Processing received message`); // Process all payloads in the envelope const payloads_list = []; // Get number of payloads const num_payloads = json_data.payloads ? json_data.payloads.length : 0; for (let i = 0; i < num_payloads; i++) { const payload = json_data.payloads[i]; const transport = payload.transport; const dataname = payload.dataname; if (transport === "direct") { // Direct transport - payload is in the message log_trace(json_data.correlation_id, `Direct transport - decoding payload '${dataname}'`); // Extract base64 payload from the payload const payload_b64 = payload.data; // Decode Base64 payload const payload_bytes = base64ToUint8Array(payload_b64); // Deserialize based on type const data_type = payload.payload_type; const data = _deserialize_data(payload_bytes, data_type, json_data.correlation_id); payloads_list.push({ dataname, data, type: data_type }); } else if (transport === "link") { // Link transport - payload is at URL const url = payload.data; log_trace(json_data.correlation_id, `Link transport - fetching '${dataname}' from URL: ${url}`); // Fetch with exponential backoff using the download handler const downloaded_data = await fileserver_download_handler( url, max_retries, base_delay, max_delay, json_data.correlation_id ); // Deserialize based on type const data_type = payload.payload_type; const data = _deserialize_data(downloaded_data, data_type, json_data.correlation_id); payloads_list.push({ dataname, data, type: data_type }); } else { throw new Error(`Unknown transport type for payload '${dataname}': ${transport}`); } } // Replace payloads array with the processed list of {dataname, data, type} tuples // This matches Julia's smartreceive return format json_data.payloads = payloads_list; return json_data; } // plik_oneshot_upload - matches Julia plik_oneshot_upload function // Upload handler signature: plik_oneshot_upload(fileserver_url, dataname, data) // Returns: { status, uploadid, fileid, url } async function plik_oneshot_upload(file_server_url, dataname, data) { /** * Upload a single file to a plik server using one-shot mode * This function uploads raw byte array to a plik server in one-shot mode (no upload session). * It first creates a one-shot upload session by sending a POST request with {"OneShot": true}, * retrieves an upload ID and token, then uploads the file data as multipart form data using the token. * * This is the default upload handler used by smartsend. * Custom handlers can be passed via the fileserver_upload_handler option. * * @param {string} file_server_url - Base URL of the plik server (e.g., "http://localhost:8080") * @param {string} dataname - Name of the file being uploaded * @param {Uint8Array} data - Raw byte data of the file content * @returns {Promise} - Dictionary with keys: status, uploadid, fileid, url */ // Step 1: Get upload ID and token const url_getUploadID = `${file_server_url}/upload`; const headers = { "Content-Type": "application/json" }; const body = JSON.stringify({ OneShot: true }); let http_response = await fetch(url_getUploadID, { method: "POST", headers: headers, body: body }); const response_json = await http_response.json(); const uploadid = response_json.id; const uploadtoken = response_json.uploadToken; // Step 2: Upload file data const url_upload = `${file_server_url}/file/${uploadid}`; // Create multipart form data const formData = new FormData(); const blob = new Blob([data], { type: "application/octet-stream" }); formData.append("file", blob, dataname); http_response = await fetch(url_upload, { method: "POST", headers: { "X-UploadToken": uploadtoken }, body: formData }); const fileResponseJson = await http_response.json(); const fileid = fileResponseJson.id; // URL of the uploaded data e.g. "http://192.168.1.20:8080/file/3F62E/4AgGT/test.zip" const url = `${file_server_url}/file/${uploadid}/${fileid}/${encodeURIComponent(dataname)}`; return { status: http_response.status, uploadid: uploadid, fileid: fileid, url: url }; } // Export for Node.js if (typeof module !== 'undefined' && module.exports) { module.exports = { MessageEnvelope, MessagePayload, smartsend, smartreceive, _serialize_data, _deserialize_data, _fetch_with_backoff, _upload_to_fileserver, plik_oneshot_upload, DEFAULT_SIZE_THRESHOLD, DEFAULT_NATS_URL, DEFAULT_FILESERVER_URL, uuid4, log_trace }; } // Export for browser if (typeof window !== 'undefined') { window.NATSBridge = { MessageEnvelope, MessagePayload, smartsend, smartreceive, _serialize_data, _deserialize_data, _fetch_with_backoff, _upload_to_fileserver, plik_oneshot_upload, DEFAULT_SIZE_THRESHOLD, DEFAULT_NATS_URL, DEFAULT_FILESERVER_URL, uuid4, log_trace }; }