From abe4e07aea427421681d61f709dcd34767851772 Mon Sep 17 00:00:00 2001 From: Akram Azarm Date: Wed, 26 Nov 2025 14:59:02 +0530 Subject: [PATCH] Publish translation logs to moesif from translator --- README.md | 25 ++- translator/integrations/ftp-sftp/README.md | 24 +++ .../ftp-sftp/swiftMtMxTranslator/Config.toml | 19 +- .../swiftMtMxTranslator/Dependencies.toml | 2 +- .../swiftMtMxTranslator/configurables.bal | 11 + .../ftp-sftp/swiftMtMxTranslator/types.bal | 11 + .../ftp-sftp/swiftMtMxTranslator/utils.bal | 188 +++++++++++++----- 7 files changed, 223 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 71e2b38..0ca7721 100644 --- a/README.md +++ b/README.md @@ -108,10 +108,31 @@ java -jar swiftMtMxTranslator.jar **Configuration:** ```bash -# Create a configuration file and configure the setup. -Refer to the sample [Config.toml](https://github.com/wso2/reference-implementation-cbpr/blob/main/translator/integrations/ftp-sftp/swiftMtMxTranslator/Config.toml) for details. +# Create a configuration file (Config.toml) and configure the setup. +# Refer to the sample Config.toml for details: +# https://github.com/wso2/reference-implementation-cbpr/blob/main/translator/integrations/ftp-sftp/swiftMtMxTranslator/Config.toml ``` +**Optional: Enable Moesif Analytics** + +To send translation events to Moesif for analytics: + +1. Sign up at [moesif.com](https://www.moesif.com/) and create an application +2. Copy your Application ID +3. Edit your `Config.toml`: + ```toml + [moesif] + enabled = true + applicationId = "your-moesif-application-id" + apiEndpoint = "https://api.moesif.net/v1/actions" + timeout = 5.0 + retryCount = 3 + ``` + +This allows the dashboard to display real-time analytics from Moesif instead of OpenSearch. + +--- + ### Option 2: Development Setup (For Contributors) **Prerequisites for Development:** diff --git a/translator/integrations/ftp-sftp/README.md b/translator/integrations/ftp-sftp/README.md index e3ae9bd..002858c 100644 --- a/translator/integrations/ftp-sftp/README.md +++ b/translator/integrations/ftp-sftp/README.md @@ -210,6 +210,30 @@ postProcess = true # Enable/disable post-processing for MT to MX translation basepath = "http://localhost:9090" # Base URL for extension APIs ``` +### Moesif Analytics (Optional) + +Send translation events to Moesif for real-time analytics: + +```toml +[moesif] +enabled = true # Enable Moesif event publishing +applicationId = "your-moesif-application-id" # From moesif.com dashboard +apiEndpoint = "https://api.moesif.net/v1/actions" # Moesif API endpoint +timeout = 5.0 # HTTP timeout in seconds +retryCount = 3 # Number of retry attempts +retryInterval = 2.0 # Delay between retries +retryBackOffFactor = 2.0 # Exponential backoff multiplier +retryMaxWaitInterval = 10.0 # Maximum wait time between retries +``` + +To enable: +1. Sign up at [moesif.com](https://www.moesif.com/) +2. Create a new application +3. Copy your Application ID +4. Set `enabled = true` and configure `applicationId` + +--- + ### Directory Structure The FTP/SFTP server must have a similar directory structure. You can change the paths in the configuration file as diff --git a/translator/integrations/ftp-sftp/swiftMtMxTranslator/Config.toml b/translator/integrations/ftp-sftp/swiftMtMxTranslator/Config.toml index cb11d21..1b4c5ee 100644 --- a/translator/integrations/ftp-sftp/swiftMtMxTranslator/Config.toml +++ b/translator/integrations/ftp-sftp/swiftMtMxTranslator/Config.toml @@ -83,7 +83,6 @@ outwardFilepath = "/mx/outward/" outputFileNamePattern = ".xml" skippedOutputFileNamePattern = ".fin" - [ballerina.log] level = "DEBUG" format="json" @@ -94,12 +93,12 @@ dashboardLogFilePath = "/swiftTranslator/logs/" # Path to the Ballerina log file ballerinaLogFilePath = "/swiftTranslator/logs/" - [translator] supportedMTMessageTypes = [ "103", "110", "111", "112", "190", "191", "192", "196", "199", "202", "205", "210", "290", "291", "292", "296", "299", "900", "910", "940", "942" ] + [translator.mxMtExtension] preProcess = true postProcess = true @@ -114,3 +113,19 @@ basepath = "http://localhost:9090" [ballerinax.financial.swiftmtToIso20022] additionalSenderToReceiverInfoCodes = ["REC"] + +# Moesif API Analytics Configuration +[moesif] +# Enable/disable Moesif event publishing (set to true to enable) +enabled = false +# Your Moesif Application ID from https://www.moesif.com/ +applicationId = "your-moesif-application-id" +# Moesif API endpoint +apiEndpoint = "https://api.moesif.net/v1/actions" +# HTTP timeout in seconds +timeout = 5.0 +# Retry configuration for failed requests +retryCount = 3 +retryInterval = 2.0 +retryBackOffFactor = 2.0 +retryMaxWaitInterval = 10.0 diff --git a/translator/integrations/ftp-sftp/swiftMtMxTranslator/Dependencies.toml b/translator/integrations/ftp-sftp/swiftMtMxTranslator/Dependencies.toml index f48abd2..d345231 100644 --- a/translator/integrations/ftp-sftp/swiftMtMxTranslator/Dependencies.toml +++ b/translator/integrations/ftp-sftp/swiftMtMxTranslator/Dependencies.toml @@ -418,7 +418,7 @@ modules = [ [[package]] org = "wso2" name = "swiftMtMxTranslator" -version = "2.0.3" +version = "2.0.5" dependencies = [ {org = "ballerina", name = "ftp"}, {org = "ballerina", name = "http"}, diff --git a/translator/integrations/ftp-sftp/swiftMtMxTranslator/configurables.bal b/translator/integrations/ftp-sftp/swiftMtMxTranslator/configurables.bal index 07f5a92..8857557 100644 --- a/translator/integrations/ftp-sftp/swiftMtMxTranslator/configurables.bal +++ b/translator/integrations/ftp-sftp/swiftMtMxTranslator/configurables.bal @@ -112,6 +112,17 @@ configurable LogConfig log = { ballerinaLogFilePath: "/logs/" }; +# Configurables for Moesif API integration +configurable MoesifConfig moesif = { + enabled: false, + applicationId: "", + apiEndpoint: "https://api.moesif.net/v1/actions", + timeout: 5.0, + retryCount: 3, + retryInterval: 2.0, + retryBackOffFactor: 2.0, + retryMaxWaitInterval: 10.0 +}; string[] & readonly supportedMTMessageTypes = [ "103", "110", "111", "112", "190", "191", "192", "196", "199", "202", "205", "210", "290", "291", "292", "296", diff --git a/translator/integrations/ftp-sftp/swiftMtMxTranslator/types.bal b/translator/integrations/ftp-sftp/swiftMtMxTranslator/types.bal index dd5a9b0..d4e70ac 100644 --- a/translator/integrations/ftp-sftp/swiftMtMxTranslator/types.bal +++ b/translator/integrations/ftp-sftp/swiftMtMxTranslator/types.bal @@ -54,6 +54,17 @@ type LogConfig record { string ballerinaLogFilePath; }; +type MoesifConfig record { + boolean enabled = false; + string applicationId; + string apiEndpoint = "https://api.moesif.net/v1/actions"; + decimal timeout = 5.0; + int retryCount = 3; + decimal retryInterval = 2.0; + float retryBackOffFactor = 2.0; + decimal retryMaxWaitInterval = 10.0; +}; + type FtpClient record { ClientConfig clientConfig; ftp:Client? 'client; diff --git a/translator/integrations/ftp-sftp/swiftMtMxTranslator/utils.bal b/translator/integrations/ftp-sftp/swiftMtMxTranslator/utils.bal index 7c9b712..a128bb2 100644 --- a/translator/integrations/ftp-sftp/swiftMtMxTranslator/utils.bal +++ b/translator/integrations/ftp-sftp/swiftMtMxTranslator/utils.bal @@ -15,12 +15,24 @@ // under the License. import ballerina/ftp; +import ballerina/http; import ballerina/io; import ballerina/log; import ballerina/regex; import ballerina/time; import ballerina/uuid; +// Initialize Moesif HTTP client (only if enabled) +final http:Client? moesifClient = moesif.enabled ? check new (moesif.apiEndpoint, { + timeout: moesif.timeout, + retryConfig: { + count: moesif.retryCount, + interval: moesif.retryInterval, + backOffFactor: moesif.retryBackOffFactor, + maxWaitInterval: moesif.retryMaxWaitInterval + } +}) : (); + # Handle error scenarios for FTP client and listener operations # # + ftpClient - FtpClient instance to interact with the FTP server @@ -35,9 +47,13 @@ function handleError(FtpClient ftpClient, string listenerName, string logId, str string fileId, string direction, string refId) { log:printInfo(string `[Listener - ${listenerName}][${logId}] Message translation failed. Sending to FTP.`); sendToSourceFTP(ftpClient, logId, FAILURE, incomingMsg, fileId); - appendToDashboardLogs(listenerName, incomingMsg, translatedMessage = NOT_AVAILABLE, msgId = fileId, refId = refId, - direction = direction, mtmsgType = UNKNOWN, mxMsgType = UNKNOWN, currency = NOT_AVAILABLE, - amount = NOT_AVAILABLE, errorMsg = errorMsg.toBalString(), status = FAILED); + + json publishableTranslationEvent = createPublishableTranslationEvent(incomingMsg, translatedMessage = NOT_AVAILABLE, + msgId = fileId, refId = refId, direction = direction, mtmsgType = UNKNOWN, mxMsgType = UNKNOWN, + currency = NOT_AVAILABLE, amount = NOT_AVAILABLE, errorMsg = errorMsg.toBalString(), status = FAILED); + + appendToDashboardLogs(listenerName, publishableTranslationEvent, fileId); + sendToMoesif(publishableTranslationEvent, listenerName, fileId); return; } @@ -76,11 +92,15 @@ function handleSkip(FtpClient sourceClient, FtpClient destinationClient, string return; } - sendToSourceFTP(sourceClient, logId, SKIP, postProcessedMsg, fileId); - sendToDestinationFTP(destinationClient, logId, postProcessedMsg, fileId, false, extension); - appendToDashboardLogs(listenerName, postProcessedMsg, translatedMessage = NOT_AVAILABLE, msgId = fileId, refId = refId, - direction = direction, mtmsgType = mtmsgType, mxMsgType = mxMsgType, currency = NOT_AVAILABLE, - amount = NOT_AVAILABLE, status = SKIPPED); + sendToSourceFTP(sourceClient, logId, SKIP, incomingMsg, fileId); + sendToDestinationFTP(destinationClient, logId, incomingMsg, fileId, false, extension); + + json publishableTranslationEvent = createPublishableTranslationEvent(incomingMsg, translatedMessage = NOT_AVAILABLE, + msgId = fileId, refId = refId, direction = direction, mtmsgType = mtmsgType, mxMsgType = mxMsgType, + currency = NOT_AVAILABLE, amount = NOT_AVAILABLE, status = SKIPPED); + + appendToDashboardLogs(listenerName, publishableTranslationEvent, fileId); + sendToMoesif(publishableTranslationEvent, listenerName, fileId); return; } @@ -106,59 +126,25 @@ function handleSuccess(FtpClient sourceClient, FtpClient destinationClient, stri log:printInfo(string `[Listener - ${listenerName}][${logId}] Message translated successfully. Sending to FTP.`); sendToSourceFTP(sourceClient, logId, SUCCESS, incomingMsg, fileId); sendToDestinationFTP(destinationClient, logId, translatedMsg, fileId); - appendToDashboardLogs(listenerName, incomingMsg, translatedMessage = translatedMsg.toBalString(), msgId = fileId, - refId = refId, direction = direction, mtmsgType = mtmsgType, mxMsgType = mxMsgType, currency = currency, - amount = amount, status = SUCCESSFUL); + + json publishableTranslationEvent = createPublishableTranslationEvent(incomingMsg, translatedMessage = translatedMsg.toBalString(), + msgId = fileId, refId = refId, direction = direction, mtmsgType = mtmsgType, mxMsgType = mxMsgType, + currency = currency, amount = amount, status = SUCCESSFUL); + + appendToDashboardLogs(listenerName, publishableTranslationEvent, fileId); + sendToMoesif(publishableTranslationEvent, listenerName, fileId); return; } # Append a log entry to the dashboard logs file. # # + listenerName - ftp listener name for logging purposes -# + orgnlMessage - original message -# + translatedMessage - translated message +# + publishableTranslationEvent - JSON publishable translation event object # + msgId - message id -# + refId - reference id -# + direction - direction of the message (inward or outward) -# + mtmsgType - MT message type -# + mxMsgType - MX message type -# + currency - currency of the transaction -# + amount - amount of the transaction -# + errorMsg - error message -# + status - status of the operation (successful, failed, skipped) -function appendToDashboardLogs(string listenerName, string orgnlMessage, string translatedMessage, string msgId, - string refId, string direction, string mtmsgType, string mxMsgType, string currency, string amount, - string errorMsg = "", string status = SUCCESSFUL) { - - // Create values for the JSON object - time:Utc currentTime = time:utcNow(); - time:Civil civilTime = time:utcToCivil(currentTime); - string|error timestamp = time:civilToString(civilTime); - string timeString = timestamp is string ? timestamp : "0000-00-00T00:00:00Z"; - - // Create the JSON log entry - json logEntry = { - "id": msgId, - "refId": refId, - "mtMessageType": mtmsgType, - "mxMessageType": mxMsgType, - "currency": currency, - "amount": amount, - "date": timeString, - "direction": direction, - "translatedMessage": translatedMessage, - "status": status, - "originalMessage": orgnlMessage, - "fieldError": errorMsg.includes("required") ? errorMsg : "", - "notSupportedError": errorMsg.toLowerAscii().includes("not supported") ? errorMsg : "", - "invalidError": errorMsg.toLowerAscii().includes("invalid") - && !errorMsg.toLowerAscii().includes("not supported") ? errorMsg : "", - "otherError": !errorMsg.includes("required") && !errorMsg.includes("not supported") - && !errorMsg.toLowerAscii().includes("invalid") ? errorMsg : "" - }; +function appendToDashboardLogs(string listenerName, json publishableTranslationEvent, string msgId) { // Convert to JSON string and append newline - string jsonLogString = logEntry.toJsonString() + "\n"; + string jsonLogString = publishableTranslationEvent.toJsonString() + "\n"; // Write to file io:FileWriteOption option = OPTION_APPEND; @@ -419,3 +405,101 @@ function extractRefId(json? mtMsgBlock4, xml? mxMsg) returns string { } return refId; } + +# Create a publishable translation event JSON object with translation details. +# +# + orgnlMessage - original message +# + translatedMessage - translated message +# + msgId - message id +# + refId - reference id +# + direction - direction of the message (inward or outward) +# + mtmsgType - MT message type +# + mxMsgType - MX message type +# + currency - currency of the transaction +# + amount - amount of the transaction +# + errorMsg - error message +# + status - status of the operation (successful, failed, skipped) +# + return - JSON publishable translation event object +function createPublishableTranslationEvent(string orgnlMessage, string translatedMessage, string msgId, string refId, + string direction, string mtmsgType, string mxMsgType, string currency, string amount, + string errorMsg = "", string status = SUCCESSFUL) returns json { + + // Create timestamp for the publishable translation event + time:Utc currentTime = time:utcNow(); + time:Civil civilTime = time:utcToCivil(currentTime); + string|error timestamp = time:civilToString(civilTime); + string timeString = timestamp is string ? timestamp : "0000-00-00T00:00:00Z"; + + // Create and return the JSON publishable translation event object + return { + "id": msgId, + "refId": refId, + "mtMessageType": mtmsgType, + "mxMessageType": mxMsgType, + "currency": currency, + "amount": amount, + "date": timeString, + "direction": direction, + "translatedMessage": translatedMessage, + "status": status, + "originalMessage": orgnlMessage, + "fieldError": errorMsg.includes("required") ? errorMsg : "", + "notSupportedError": errorMsg.toLowerAscii().includes("not supported") ? errorMsg : "", + "invalidError": errorMsg.toLowerAscii().includes("invalid") + && !errorMsg.toLowerAscii().includes("not supported") ? errorMsg : "", + "otherError": !errorMsg.includes("required") && !errorMsg.includes("not supported") + && !errorMsg.toLowerAscii().includes("invalid") ? errorMsg : "" + }; +} + + +# Send translation event to Moesif API asynchronously. +# This function transforms the publishable translation event into Moesif's action event format and sends it +# using a worker thread to avoid blocking the main execution flow. +# +# + publishableTranslationEvent - the JSON publishable translation event object +# + listenerName - listener name for error logging +# + msgId - message ID for tracking +function sendToMoesif(json publishableTranslationEvent, string listenerName, string msgId) { + + // Skip if Moesif is disabled or client not initialized + if !moesif.enabled || moesifClient is () { + return; + } + + // Send event asynchronously using a worker (non-blocking) + worker MoesifWorker { + do { + // Extract status for action name + json statusField = check publishableTranslationEvent.status; + string statusValue = statusField is string ? statusField : ""; + + // Transform to Moesif action format + json moesifAction = { + "action_name": "translation_log", + "request": { + "time": check publishableTranslationEvent.date + }, + "metadata": publishableTranslationEvent + }; + + // Send to Moesif API + http:Client clientRef = moesifClient; + http:Request request = new; + request.setHeader("X-Moesif-Application-Id", moesif.applicationId); + request.setHeader("Content-Type", "application/json"); + request.setJsonPayload(moesifAction); + + http:Response|http:ClientError response = clientRef->post("", request); + if response is http:ClientError { + log:printError(string `[Listener - ${listenerName}][${msgId}] Failed to send event to Moesif`, + err = response.toBalString()); + } else { + log:printDebug(string `[Listener - ${listenerName}][${msgId}] Event sent to Moesif. Status: ${response.statusCode}`); + } + } on fail error e { + log:printError(string `[Listener - ${listenerName}][${msgId}] Error processing Moesif event`, + err = e.toBalString()); + } + } +}