diff --git a/.changeset/warm-hornets-compete.md b/.changeset/warm-hornets-compete.md new file mode 100644 index 0000000000..bd1ca21dd4 --- /dev/null +++ b/.changeset/warm-hornets-compete.md @@ -0,0 +1,5 @@ +--- +'@chainlink/ftse-sftp-adapter': major +--- + +Adding Downloading and parsing logic for russell and ftse csv files from ftse sftp server diff --git a/packages/sources/ftse-sftp/src/index.ts b/packages/sources/ftse-sftp/src/index.ts index c458ca757c..b72a268b6d 100644 --- a/packages/sources/ftse-sftp/src/index.ts +++ b/packages/sources/ftse-sftp/src/index.ts @@ -1 +1,12 @@ -export * from './parsing' +import { expose, ServerInstance } from '@chainlink/external-adapter-framework' +import { Adapter } from '@chainlink/external-adapter-framework/adapter' +import { config } from './config' +import * as endpoints from './endpoint' + +export const adapter = new Adapter({ + name: 'FTSE_SFTP', + config, + endpoints: [endpoints.sftp.endpoint], +}) + +export const server = (): Promise => expose(adapter) diff --git a/packages/sources/ftse-sftp/src/transport/constants.ts b/packages/sources/ftse-sftp/src/transport/constants.ts new file mode 100644 index 0000000000..e958505613 --- /dev/null +++ b/packages/sources/ftse-sftp/src/transport/constants.ts @@ -0,0 +1,29 @@ +export const instrumentToFilePathMap: Record = { + FTSE100INDEX: '/data/valuation/uk_all_share/', + Russell1000INDEX: '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/', + Russell2000INDEX: '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/', + Russell3000INDEX: '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/', +} + +export const instrumentToFileTemplateMap: Record = { + FTSE100INDEX: 'ukallv*.csv', + Russell1000INDEX: 'daily_values_russell_*.CSV', + Russell2000INDEX: 'daily_values_russell_*.CSV', + Russell3000INDEX: 'daily_values_russell_*.CSV', +} + +export const instrumentToFileRegexMap: Record = { + FTSE100INDEX: /^ukallv\d{4}\.csv$/, + Russell1000INDEX: /^daily_values_russell_\d{6}\.CSV$/, + Russell2000INDEX: /^daily_values_russell_\d{6}\.CSV$/, + Russell3000INDEX: /^daily_values_russell_\d{6}\.CSV$/, +} + +/** + * Validates if an instrument is supported by checking if it has all required mappings + * @param instrument The instrument identifier to validate + * @returns true if the instrument is supported, false otherwise + */ +export function isInstrumentSupported(instrument: string): boolean { + return !!(instrumentToFilePathMap[instrument] && instrumentToFileRegexMap[instrument]) +} diff --git a/packages/sources/ftse-sftp/src/transport/sftp.ts b/packages/sources/ftse-sftp/src/transport/sftp.ts index c5860bbb34..37b6324e5d 100644 --- a/packages/sources/ftse-sftp/src/transport/sftp.ts +++ b/packages/sources/ftse-sftp/src/transport/sftp.ts @@ -1,13 +1,35 @@ import { EndpointContext } from '@chainlink/external-adapter-framework/adapter' +import { ResponseCache } from '@chainlink/external-adapter-framework/cache/response' import { TransportDependencies } from '@chainlink/external-adapter-framework/transports' import { SubscriptionTransport } from '@chainlink/external-adapter-framework/transports/abstract/subscription' -import { sleep } from '@chainlink/external-adapter-framework/util' -import SftpClient from 'ssh2-sftp-client' -import { BaseEndpointTypes } from '../endpoint/sftp' +import { AdapterResponse, makeLogger, sleep } from '@chainlink/external-adapter-framework/util' +import { AdapterInputError } from '@chainlink/external-adapter-framework/validation/error' +import SftpClient, { FileInfo } from 'ssh2-sftp-client' +import { BaseEndpointTypes, IndexResponseData, inputParameters } from '../endpoint/sftp' +import { CSVParserFactory } from '../parsing/factory' +import { + instrumentToFilePathMap, + instrumentToFileRegexMap, + isInstrumentSupported, +} from './constants' + +const logger = makeLogger('FTSE SFTP Adapter') + +type RequestParams = typeof inputParameters.validated + +interface SftpConnectionConfig { + host: string + port: number + username: string + password: string + readyTimeout: number +} export class SftpTransport extends SubscriptionTransport { config!: BaseEndpointTypes['Settings'] endpointName!: string + name!: string + responseCache!: ResponseCache sftpClient: SftpClient constructor() { @@ -24,12 +46,156 @@ export class SftpTransport extends SubscriptionTransport { await super.initialize(dependencies, adapterSettings, endpointName, transportName) this.config = adapterSettings this.endpointName = endpointName + this.name = transportName + this.responseCache = dependencies.responseCache } - async backgroundHandler(context: EndpointContext): Promise { + async backgroundHandler( + context: EndpointContext, + entries: RequestParams[], + ): Promise { + await Promise.all(entries.map(async (param) => this.handleRequest(param))) await sleep(context.adapterSettings.BACKGROUND_EXECUTE_MS) } + async handleRequest(param: RequestParams) { + let response: AdapterResponse + try { + response = await this._handleRequest(param) + } catch (e) { + const errorMessage = e instanceof Error ? e.message : 'Unknown error occurred' + response = { + statusCode: 502, + errorMessage, + timestamps: { + providerDataRequestedUnixMs: 0, + providerDataReceivedUnixMs: 0, + providerIndicatedTimeUnixMs: undefined, + }, + } + } finally { + try { + await this.sftpClient.end() + logger.info('SFTP connection closed') + } catch (error) { + logger.error('Error closing SFTP connection:', error) + } + } + + await this.responseCache.write(this.name, [{ params: param, response }]) + } + + async _handleRequest( + param: RequestParams, + ): Promise> { + const providerDataRequestedUnixMs = Date.now() + + await this.connectToSftp() + + const parsedData = await this.tryDownloadAndParseFile(param.instrument) + + // Extract the numeric result based on the data type + let result: number + if ('gbpIndex' in parsedData) { + // FTSE data + result = (parsedData.gbpIndex as number) ?? 0 + } else if ('close' in parsedData) { + // Russell data + result = parsedData.close as number + } else { + throw new Error('Unknown data format received from parser') + } + + logger.info(`Successfully processed data for instrument: ${param.instrument}`) + return { + data: { + result: parsedData, + }, + statusCode: 200, + result, + timestamps: { + providerDataRequestedUnixMs, + providerDataReceivedUnixMs: Date.now(), + providerIndicatedTimeUnixMs: undefined, + }, + } + } + + private async connectToSftp(): Promise { + const connectConfig: SftpConnectionConfig = { + host: this.config.SFTP_HOST, + port: this.config.SFTP_PORT || 22, + username: this.config.SFTP_USERNAME, + password: this.config.SFTP_PASSWORD, + readyTimeout: 30000, + } + + try { + // Create a new client instance to avoid connection state issues + this.sftpClient = new SftpClient() + await this.sftpClient.connect(connectConfig) + logger.info('Successfully connected to SFTP server') + } catch (error) { + logger.error(error, 'Failed to connect to SFTP server') + throw new AdapterInputError({ + statusCode: 500, + message: `Failed to connect to SFTP server: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + }) + } + } + + private async tryDownloadAndParseFile(instrument: string): Promise { + // Validate that the instrument is supported + if (!isInstrumentSupported(instrument)) { + throw new AdapterInputError({ + statusCode: 400, + message: `Unsupported instrument: ${instrument}`, + }) + } + + const filePath = instrumentToFilePathMap[instrument] + const fileRegex = instrumentToFileRegexMap[instrument] + + const fileList = await this.sftpClient.list(filePath) + // Filter files based on the regex pattern + const matchingFiles = fileList + .map((file: FileInfo) => file.name) + .filter((fileName: string) => fileRegex.test(fileName)) + + if (matchingFiles.length === 0) { + throw new AdapterInputError({ + statusCode: 500, + message: `No files matching pattern ${fileRegex} found in directory: ${filePath}`, + }) + } else if (matchingFiles.length > 1) { + throw new AdapterInputError({ + statusCode: 500, + message: `Multiple files matching pattern ${fileRegex} found in directory: ${filePath}.`, + }) + } + const fullPath = `${filePath}${matchingFiles[0]}` + + // Log the download attempt + logger.info(`Downloading file: ${fullPath}`) + + const fileContent = await this.sftpClient.get(fullPath) + // we need latin1 here because the file contains special characters like "®" + const csvContent = fileContent.toString('latin1') + + const parser = CSVParserFactory.detectParserByInstrument(instrument) + + if (!parser) { + throw new AdapterInputError({ + statusCode: 500, + message: `Parser initialization failed for instrument: ${instrument}`, + }) + } + + return (await parser.parse(csvContent)) as IndexResponseData + } + getSubscriptionTtlFromConfig(adapterSettings: BaseEndpointTypes['Settings']): number { return adapterSettings.BACKGROUND_EXECUTE_MS || 60000 } diff --git a/packages/sources/ftse-sftp/test/integration/__snapshots__/adapter.test.ts.snap b/packages/sources/ftse-sftp/test/integration/__snapshots__/adapter.test.ts.snap new file mode 100644 index 0000000000..29ae820f14 --- /dev/null +++ b/packages/sources/ftse-sftp/test/integration/__snapshots__/adapter.test.ts.snap @@ -0,0 +1,72 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`execute ftse_sftp endpoint should return success for FTSE100INDEX 1`] = ` +{ + "data": { + "result": { + "gbpIndex": 8045.12345678, + "indexBaseCurrency": "GBP", + "indexCode": "UKX", + "indexSectorName": "FTSE 100 Index", + "numberOfConstituents": 100, + }, + }, + "result": 8045.12345678, + "statusCode": 200, + "timestamps": { + "providerDataReceivedUnixMs": 1641035471111, + "providerDataRequestedUnixMs": 1641035471111, + }, +} +`; + +exports[`execute ftse_sftp endpoint should return success for Russell1000INDEX 1`] = ` +{ + "data": { + "result": { + "close": 2654.123456, + "indexName": "Russell 1000® Index", + }, + }, + "result": 2654.123456, + "statusCode": 200, + "timestamps": { + "providerDataReceivedUnixMs": 1641035471111, + "providerDataRequestedUnixMs": 1641035471111, + }, +} +`; + +exports[`execute ftse_sftp endpoint should return success for Russell2000INDEX 1`] = ` +{ + "data": { + "result": { + "close": 1987.654321, + "indexName": "Russell 2000® Index", + }, + }, + "result": 1987.654321, + "statusCode": 200, + "timestamps": { + "providerDataReceivedUnixMs": 1641035471111, + "providerDataRequestedUnixMs": 1641035471111, + }, +} +`; + +exports[`execute ftse_sftp endpoint should return success for Russell3000INDEX 1`] = ` +{ + "data": { + "result": { + "close": 3456.789012, + "indexName": "Russell 3000® Index", + }, + }, + "result": 3456.789012, + "statusCode": 200, + "timestamps": { + "providerDataReceivedUnixMs": 1641035471111, + "providerDataRequestedUnixMs": 1641035471111, + }, +} +`; diff --git a/packages/sources/ftse-sftp/test/integration/adapter.test.ts b/packages/sources/ftse-sftp/test/integration/adapter.test.ts new file mode 100644 index 0000000000..6fc439af5e --- /dev/null +++ b/packages/sources/ftse-sftp/test/integration/adapter.test.ts @@ -0,0 +1,299 @@ +import { Adapter } from '@chainlink/external-adapter-framework/adapter' +import { AdapterResponse } from '@chainlink/external-adapter-framework/util' +import { + TestAdapter, + setEnvVariables, +} from '@chainlink/external-adapter-framework/util/testing-utils' +import { config } from '../../src/config' +import { BaseEndpointTypes, IndexResponseData } from '../../src/endpoint/sftp' +import { + mockFtse100Success, + mockRussell1000Success, + mockRussell2000Success, + mockRussell3000Success, +} from './fixtures' + +// Import the actual types from source files +type AdapterSettings = typeof config.settings + +// Define types for better type safety +type MockSftpClient = { + connect: jest.MockedFunction<() => Promise> + end: jest.MockedFunction<() => Promise> + get: jest.MockedFunction<() => Promise> + fastGet: jest.MockedFunction<() => Promise> + exists: jest.MockedFunction<() => Promise> + list: jest.MockedFunction<() => Promise>> + stat: jest.MockedFunction<() => Promise>> +} + +interface MockAdapterHandleRequest { + statusCode: number + result: number // This should be the extracted numeric value + data: { result: IndexResponseData } + timestamps: { + providerDataRequestedUnixMs: number + providerDataReceivedUnixMs: number + providerIndicatedTimeUnixMs: number | undefined + } +} + +interface MockSftpTransportDependencies { + responseCache?: { + write: ( + name: string, + data: Array<{ params: { instrument: string }; response: MockAdapterHandleRequest }>, + ) => Promise + } +} + +interface MockSftpTransportContext { + adapterSettings: AdapterSettings +} + +interface MockSftpTransport { + name: string + config: Record + responseCache: { + write?: ( + name: string, + data: Array<{ params: { instrument: string }; response: MockAdapterHandleRequest }>, + ) => Promise + } + endpointName: string + initialize( + dependencies: MockSftpTransportDependencies, + adapterSettings: AdapterSettings, + endpointName: string, + transportName: string, + ): Promise + backgroundHandler( + context: MockSftpTransportContext, + entries: Array<{ instrument: string }>, + ): Promise + processRequest(param: { instrument: string }): Promise + getSubscriptionTtlFromConfig(): number +} + +// Mock the entire SFTP module to avoid any actual SFTP connections +const mockSftpClient: MockSftpClient = { + connect: jest.fn().mockResolvedValue(undefined), + end: jest.fn().mockResolvedValue(undefined), + get: jest.fn(), + fastGet: jest.fn().mockResolvedValue(Buffer.from('')), + exists: jest.fn().mockResolvedValue(true), + list: jest.fn(), + stat: jest.fn().mockResolvedValue({}), +} + +jest.mock('ssh2-sftp-client', () => { + return jest.fn().mockImplementation(() => mockSftpClient) +}) + +// Create a more realistic mock of the SFTP transport that actually processes requests +jest.mock('../../src/transport/sftp', () => { + const originalModule = jest.requireActual('../../src/transport/sftp') + + return { + ...originalModule, + sftpTransport: { + name: 'default_single_transport', + config: {} as Record, + responseCache: {} as Record, + endpointName: '', + async initialize( + dependencies: MockSftpTransportDependencies, + adapterSettings: AdapterSettings, + endpointName: string, + transportName: string, + ) { + this.config = adapterSettings as Record + this.endpointName = endpointName + this.name = transportName + this.responseCache = dependencies.responseCache || {} + }, + async backgroundHandler( + _context: MockSftpTransportContext, + entries: Array<{ instrument: string }>, + ) { + // Process each entry and write to cache + for (const entry of entries) { + const result = await this.processRequest(entry) + if (this.responseCache && this.responseCache.write) { + await this.responseCache.write(this.name, [{ params: entry, response: result }]) + } + } + }, + async processRequest(param: { instrument: string }): Promise { + // Mock the successful processing based on instrument + const mockResults: Record = { + FTSE100INDEX: mockFtse100Success(), + Russell1000INDEX: mockRussell1000Success(), + Russell2000INDEX: mockRussell2000Success(), + Russell3000INDEX: mockRussell3000Success(), + } + + const result = mockResults[param.instrument] + if (!result) { + throw new Error(`Unsupported instrument: ${param.instrument}`) + } + + // Extract the numeric result based on the data type + let numericResult: number + if ('gbpIndex' in result) { + // FTSE data + const gbpValue = result.gbpIndex + numericResult = typeof gbpValue === 'number' ? gbpValue : Number(gbpValue) || 0 + } else if ('close' in result) { + // Russell data + numericResult = result.close + } else { + throw new Error('Unknown data format received from parser') + } + + return { + statusCode: 200, + data: { result }, + result: numericResult, + timestamps: { + providerDataRequestedUnixMs: Date.now(), + providerDataReceivedUnixMs: Date.now(), + providerIndicatedTimeUnixMs: undefined, + }, + } + }, + getSubscriptionTtlFromConfig() { + return 60000 + }, + } as MockSftpTransport, + } +}) + +describe('execute', () => { + let spy: jest.SpyInstance + let testAdapter: TestAdapter + let oldEnv: NodeJS.ProcessEnv + + beforeAll(async () => { + oldEnv = JSON.parse(JSON.stringify(process.env)) + process.env['SFTP_HOST'] = 'sftp.test.com' + process.env['SFTP_PORT'] = '22' + process.env['SFTP_USERNAME'] = 'testuser' + process.env['SFTP_PASSWORD'] = 'testpass' + + const mockDate = new Date('2022-01-01T11:11:11.111Z') + spy = jest.spyOn(Date, 'now').mockReturnValue(mockDate.getTime()) + + const adapter = (await import('./../../src')).adapter as unknown as Adapter + adapter.rateLimiting = undefined + testAdapter = await TestAdapter.startWithMockedCache(adapter, { + testAdapter: {} as TestAdapter, + }) + }) + + afterAll(async () => { + setEnvVariables(oldEnv) + if (testAdapter?.api?.close) { + await testAdapter.api.close() + } + spy.mockRestore() + }) + + describe('ftse_sftp endpoint', () => { + it('should return success for FTSE100INDEX', async () => { + const data = { + endpoint: 'ftse_sftp', + instrument: 'FTSE100INDEX', + } + + // Mock the response cache directly + const mockResult = mockFtse100Success() + jest.spyOn(testAdapter.adapter, 'handleRequest').mockResolvedValueOnce({ + statusCode: 200, + result: mockResult.gbpIndex, // Extract the numeric value + data: { result: mockResult }, + timestamps: { + providerDataRequestedUnixMs: Date.now(), + providerDataReceivedUnixMs: Date.now(), + providerIndicatedTimeUnixMs: undefined, + }, + } as AdapterResponse) + + const response = await testAdapter.request(data) + expect(response.statusCode).toBe(200) + expect(response.json()).toMatchSnapshot() + }) + + it('should return success for Russell1000INDEX', async () => { + const data = { + endpoint: 'ftse_sftp', + instrument: 'Russell1000INDEX', + } + + // Mock the response cache directly + const mockResult = mockRussell1000Success() + jest.spyOn(testAdapter.adapter, 'handleRequest').mockResolvedValueOnce({ + statusCode: 200, + result: mockResult.close, // Extract the numeric value + data: { result: mockResult }, + timestamps: { + providerDataRequestedUnixMs: Date.now(), + providerDataReceivedUnixMs: Date.now(), + providerIndicatedTimeUnixMs: undefined, + }, + } as AdapterResponse) + + const response = await testAdapter.request(data) + expect(response.statusCode).toBe(200) + expect(response.json()).toMatchSnapshot() + }) + + it('should return success for Russell2000INDEX', async () => { + const data = { + endpoint: 'ftse_sftp', + instrument: 'Russell2000INDEX', + } + + // Mock the response cache directly + const mockResult = mockRussell2000Success() + jest.spyOn(testAdapter.adapter, 'handleRequest').mockResolvedValueOnce({ + statusCode: 200, + result: mockResult.close, // Extract the numeric value + data: { result: mockResult }, + timestamps: { + providerDataRequestedUnixMs: Date.now(), + providerDataReceivedUnixMs: Date.now(), + providerIndicatedTimeUnixMs: undefined, + }, + } as AdapterResponse) + + const response = await testAdapter.request(data) + expect(response.statusCode).toBe(200) + expect(response.json()).toMatchSnapshot() + }) + + it('should return success for Russell3000INDEX', async () => { + const data = { + endpoint: 'ftse_sftp', + instrument: 'Russell3000INDEX', + } + + // Mock the response cache directly + const mockResult = mockRussell3000Success() + jest.spyOn(testAdapter.adapter, 'handleRequest').mockResolvedValueOnce({ + statusCode: 200, + result: mockResult.close, // Extract the numeric value + data: { result: mockResult }, + timestamps: { + providerDataRequestedUnixMs: Date.now(), + providerDataReceivedUnixMs: Date.now(), + providerIndicatedTimeUnixMs: undefined, + }, + } as AdapterResponse) + + const response = await testAdapter.request(data) + expect(response.statusCode).toBe(200) + expect(response.json()).toMatchSnapshot() + }) + }) +}) diff --git a/packages/sources/ftse-sftp/test/integration/fixtures.ts b/packages/sources/ftse-sftp/test/integration/fixtures.ts new file mode 100644 index 0000000000..cfe3170647 --- /dev/null +++ b/packages/sources/ftse-sftp/test/integration/fixtures.ts @@ -0,0 +1,60 @@ +// Mock SFTP data responses - simulating what would be returned after parsing CSV files +export const mockFtse100Response = { + indexCode: 'UKX', + indexSectorName: 'FTSE 100 Index', + numberOfConstituents: 100, + indexBaseCurrency: 'GBP', + gbpIndex: 8045.12345678, +} + +export const mockRussell1000Response = { + indexName: 'Russell 1000® Index', + close: 2654.123456, +} + +export const mockRussell2000Response = { + indexName: 'Russell 2000® Index', + close: 1987.654321, +} + +export const mockRussell3000Response = { + indexName: 'Russell 3000® Index', + close: 3456.789012, +} + +// Raw CSV fixtures for transport testing +export const mockFtseCsvContent = `27/08/2025 (C) FTSE International Limited 2025. All Rights Reserved +FTSE UK All-Share Indices Valuation Service + +Index Code,Index/Sector Name,Number of Constituents,Index Base Currency,USD Index,GBP Index,EUR Index,JPY Index,AUD Index,CNY Index,HKD Index,CAD Index,LOC Index,Base Currency (GBP) Index +UKX,FTSE 100 Index,100,GBP,4659.89484111,8045.12345678,4523.90007694,2963.46786723,6470.75900926,10384.47293100,4667.43880552,5177.36970414,,8045.12345678 +AS0,FTSE All-Small Index,234,GBP,4659.78333168,5017.12840249,4523.79182181,2963.39695263,6470.60416658,10384.22443471,4667.32711557,5177.24581174,,5017.12840249` + +export const mockRussellCsvContent = `"Daily Values",,,,,,,,,,,,,, +,,,,,,,,,,,,,, +,,,,,,,,,,,,,, +"As of August 27, 2025",,,,,,,,"Last 5 Trading Days",,,,"1 Year Ending",, +,,,,,,,,"Closing Values",,,,"Closing Values",, +,"Open","High","Low","Close","Net Chg","% Chg","High","Low","Net Chg","% Chg","High","Low","Net Chg","% Chg" +Russell 1000® Index,3538.25,3550.79,3534.60,2654.123456,9.16,0.26,3547.40,3483.25,51.20,1.46,3547.40,2719.99,496.76,16.28 +Russell 2000® Index,2358.60,2375.71,2352.78,1987.654321,15.20,0.64,2373.80,2274.10,104.45,4.60,2442.03,1760.71,185.16,8.46 +Russell 3000® Index,3680.78,3694.23,3676.84,3456.789012,10.14,0.28,3690.93,3620.34,58.02,1.60,3690.93,2826.03,506.36,15.90` + +// Mock functions to set up the SFTP responses +export const mockFtse100Success = () => { + // Since this is SFTP, we mock at the transport level rather than HTTP + // The actual mocking happens in the test file with jest.mock + return mockFtse100Response +} + +export const mockRussell1000Success = () => { + return mockRussell1000Response +} + +export const mockRussell2000Success = () => { + return mockRussell2000Response +} + +export const mockRussell3000Success = () => { + return mockRussell3000Response +} diff --git a/packages/sources/ftse-sftp/test/integration/sftp-transport.test.ts b/packages/sources/ftse-sftp/test/integration/sftp-transport.test.ts new file mode 100644 index 0000000000..b6aeda128e --- /dev/null +++ b/packages/sources/ftse-sftp/test/integration/sftp-transport.test.ts @@ -0,0 +1,453 @@ +import { EndpointContext } from '@chainlink/external-adapter-framework/adapter' +import { TransportDependencies } from '@chainlink/external-adapter-framework/transports' +import { makeStub } from '@chainlink/external-adapter-framework/util/testing-utils' +import fs from 'fs' +import path from 'path' +import { BaseEndpointTypes } from '../../src/endpoint/sftp' + +// Load actual CSV fixture files +const loadFixtureFile = (filename: string): string => { + const fixturePath = path.join(__dirname, '..', 'fixtures', filename) + return fs.readFileSync(fixturePath, 'latin1') // Use latin1 to match the actual file encoding +} + +// Load real CSV content from fixtures +const ftseFixtureContent = loadFixtureFile('ftse100.csv') +const russellFixtureContent = loadFixtureFile('daily_russell_values.CSV') + +// Define interfaces for type safety +interface MockResponseCache { + write: jest.MockedFunction< + ( + transportName: string, + results: Array<{ params: { instrument: string }; response: BaseEndpointTypes['Response'] }>, + ) => Promise + > +} + +interface MockSftpClientMethods { + connect: jest.MockedFunction<() => Promise> + list: jest.MockedFunction<() => Promise>> + get: jest.MockedFunction<() => Promise> + end: jest.MockedFunction<() => Promise> +} + +interface MockContext { + adapterSettings: BaseEndpointTypes['Settings'] +} + +// Mock the framework dependencies +jest.mock('@chainlink/external-adapter-framework/transports/abstract/subscription', () => ({ + SubscriptionTransport: class MockSubscriptionTransport { + responseCache: MockResponseCache = { + write: jest.fn(), + } + name = 'test' + config = {} + endpointName = '' + constructor() { + // Mock constructor + } + async initialize( + dependencies: TransportDependencies, + adapterSettings: BaseEndpointTypes['Settings'], + endpointName: string, + transportName: string, + ) { + this.config = adapterSettings + this.endpointName = endpointName + this.name = transportName + this.responseCache = dependencies.responseCache as unknown as MockResponseCache + } + }, +})) + +jest.mock('@chainlink/external-adapter-framework/util', () => ({ + makeLogger: jest.fn(() => ({ + info: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + })), + sleep: jest.fn(), +})) + +// Mock the ssh2-sftp-client +jest.mock('ssh2-sftp-client', () => { + const mockClientMethods = { + connect: jest.fn(), + list: jest.fn(), + get: jest.fn(), + end: jest.fn(), + } + + const MockSftpClient = jest.fn().mockImplementation(() => mockClientMethods) + // Expose the mock methods for testing + ;(MockSftpClient as any).mockClientMethods = mockClientMethods + + return MockSftpClient +}) + +// Import after mocking +import SftpClient from 'ssh2-sftp-client' +import { SftpTransport } from '../../src/transport/sftp' + +describe('SftpTransport Integration Tests', () => { + let transport: SftpTransport + let mockResponseCache: MockResponseCache + let mockDependencies: TransportDependencies + let mockAdapterSettings: BaseEndpointTypes['Settings'] + let mockSftpClient: MockSftpClientMethods + + beforeEach(async () => { + // Reset all mocks + jest.clearAllMocks() + + // Mock response cache + mockResponseCache = { + write: jest.fn(), + } + + mockDependencies = makeStub('dependencies', { + responseCache: mockResponseCache, + } as unknown as TransportDependencies) + + mockAdapterSettings = makeStub('adapterSettings', { + SFTP_HOST: 'test-sftp.example.com', + SFTP_PORT: 22, + SFTP_USERNAME: 'testuser', + SFTP_PASSWORD: 'testpass', + BACKGROUND_EXECUTE_MS: 60000, + } as unknown as BaseEndpointTypes['Settings']) + + transport = new SftpTransport() + await transport.initialize( + mockDependencies, + mockAdapterSettings, + 'ftse_sftp', + 'default_single_transport', + ) + + // Use the shared mock methods from the mocked constructor + mockSftpClient = (SftpClient as unknown as { mockClientMethods: MockSftpClientMethods }) + .mockClientMethods + + // Set up default behavior + mockSftpClient.connect.mockResolvedValue(undefined) + mockSftpClient.list.mockResolvedValue([]) + mockSftpClient.get.mockResolvedValue(Buffer.from('', 'latin1')) + mockSftpClient.end.mockResolvedValue(undefined) + }) + + describe('successful file download and parsing', () => { + it('should successfully process FTSE100INDEX data', async () => { + // Set up SFTP operations to return real fixture data + mockSftpClient.list.mockResolvedValue([{ name: 'ukallv2025.csv', size: 1000 }]) + mockSftpClient.get.mockResolvedValue(Buffer.from(ftseFixtureContent, 'latin1')) + + await transport.handleRequest({ instrument: 'FTSE100INDEX' }) + + expect(mockSftpClient.connect).toHaveBeenCalledWith({ + host: 'test-sftp.example.com', + port: 22, + username: 'testuser', + password: 'testpass', + readyTimeout: 30000, + }) + expect(mockSftpClient.list).toHaveBeenCalledWith('/data/valuation/uk_all_share/') + expect(mockSftpClient.get).toHaveBeenCalledWith('/data/valuation/uk_all_share/ukallv2025.csv') + + // Verify the response was written to cache with real parsed data + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 200, + data: expect.objectContaining({ + result: expect.objectContaining({ + indexCode: 'UKX', + indexSectorName: 'FTSE 100 Index', + numberOfConstituents: 100, + indexBaseCurrency: 'GBP', + gbpIndex: 9116.68749114, + }), + }), + }), + }, + ]) + }) + + it('should successfully process Russell1000INDEX data', async () => { + // Set up SFTP operations to return real fixture data + mockSftpClient.list.mockResolvedValue([ + { name: 'daily_values_russell_250827.CSV', size: 2000 }, + ]) + mockSftpClient.get.mockResolvedValue(Buffer.from(russellFixtureContent, 'latin1')) + + await transport.handleRequest({ instrument: 'Russell1000INDEX' }) + + expect(mockSftpClient.list).toHaveBeenCalledWith( + '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/', + ) + expect(mockSftpClient.get).toHaveBeenCalledWith( + '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/daily_values_russell_250827.CSV', + ) + + // Verify the response was written to cache with real parsed data + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'Russell1000INDEX' }, + response: expect.objectContaining({ + statusCode: 200, + data: expect.objectContaining({ + result: expect.objectContaining({ + indexName: 'Russell 1000® Index', + close: 3547.4, + }), + }), + }), + }, + ]) + }) + + it('should successfully process Russell2000INDEX data', async () => { + // Set up SFTP operations to return real fixture data + mockSftpClient.list.mockResolvedValue([ + { name: 'daily_values_russell_250827.CSV', size: 2000 }, + ]) + mockSftpClient.get.mockResolvedValue(Buffer.from(russellFixtureContent, 'latin1')) + + await transport.handleRequest({ instrument: 'Russell2000INDEX' }) + + // Verify the response was written to cache with real parsed data + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'Russell2000INDEX' }, + response: expect.objectContaining({ + statusCode: 200, + data: expect.objectContaining({ + result: expect.objectContaining({ + indexName: 'Russell 2000® Index', + close: 2373.8, + }), + }), + }), + }, + ]) + }) + + it('should successfully process Russell3000INDEX data', async () => { + // Set up SFTP operations to return real fixture data + mockSftpClient.list.mockResolvedValue([ + { name: 'daily_values_russell_250827.CSV', size: 2000 }, + ]) + mockSftpClient.get.mockResolvedValue(Buffer.from(russellFixtureContent, 'latin1')) + + await transport.handleRequest({ instrument: 'Russell3000INDEX' }) + + // Verify the response was written to cache with real parsed data + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'Russell3000INDEX' }, + response: expect.objectContaining({ + statusCode: 200, + data: expect.objectContaining({ + result: expect.objectContaining({ + indexName: 'Russell 3000® Index', + close: 3690.93, + }), + }), + }), + }, + ]) + }) + }) + + describe('error scenarios', () => { + it('should handle SFTP connection failure', async () => { + mockSftpClient.connect.mockRejectedValue(new Error('Connection timeout')) + + await transport.handleRequest({ instrument: 'FTSE100INDEX' }) + + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 502, + errorMessage: 'Failed to connect to SFTP server: Connection timeout', + }), + }, + ]) + }) + + it('should handle no matching files found', async () => { + mockSftpClient.list.mockResolvedValue([{ name: 'wrongfile.txt', size: 100 }]) + + await transport.handleRequest({ instrument: 'FTSE100INDEX' }) + + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 502, + errorMessage: + 'No files matching pattern /^ukallv\\d{4}\\.csv$/ found in directory: /data/valuation/uk_all_share/', + }), + }, + ]) + }) + + it('should handle multiple matching files found', async () => { + mockSftpClient.list.mockResolvedValue([ + { name: 'ukallv2025.csv', size: 1000 }, + { name: 'ukallv2024.csv', size: 1000 }, + ]) + + await transport.handleRequest({ instrument: 'FTSE100INDEX' }) + + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 502, + errorMessage: + 'Multiple files matching pattern /^ukallv\\d{4}\\.csv$/ found in directory: /data/valuation/uk_all_share/.', + }), + }, + ]) + }) + + it('should handle file download failure', async () => { + mockSftpClient.list.mockResolvedValue([{ name: 'ukallv2025.csv', size: 1000 }]) + mockSftpClient.get.mockRejectedValue(new Error('File not accessible')) + + await transport.handleRequest({ instrument: 'FTSE100INDEX' }) + + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 502, + errorMessage: 'File not accessible', + }), + }, + ]) + }) + + it('should handle unsupported instrument', async () => { + mockSftpClient.list.mockResolvedValue([{ name: 'somefile.csv', size: 1000 }]) + mockSftpClient.get.mockResolvedValue(Buffer.from('some content', 'latin1')) + + await transport.handleRequest({ instrument: 'UNSUPPORTED_INSTRUMENT' as 'FTSE100INDEX' }) + + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'UNSUPPORTED_INSTRUMENT' }, + response: expect.objectContaining({ + statusCode: 502, + errorMessage: 'Unsupported instrument: UNSUPPORTED_INSTRUMENT', + }), + }, + ]) + }) + }) + + describe('background handler', () => { + it('should process multiple requests in background handler', async () => { + // Set up SFTP operations for multiple instruments + mockSftpClient.list + .mockResolvedValueOnce([{ name: 'ukallv2025.csv', size: 1000 }]) + .mockResolvedValueOnce([{ name: 'daily_values_russell_250827.CSV', size: 2000 }]) + mockSftpClient.get + .mockResolvedValueOnce(Buffer.from(ftseFixtureContent, 'latin1')) + .mockResolvedValueOnce(Buffer.from(russellFixtureContent, 'latin1')) + + const mockContext: MockContext = { + adapterSettings: mockAdapterSettings, + } + + const entries: Array<{ instrument: string }> = [ + { instrument: 'FTSE100INDEX' }, + { instrument: 'Russell1000INDEX' }, + ] + + await transport.backgroundHandler( + mockContext as unknown as EndpointContext, + entries, + ) + + // Verify both requests were processed with real parsing + expect(mockResponseCache.write).toHaveBeenCalledTimes(2) + expect(mockResponseCache.write).toHaveBeenNthCalledWith(1, 'default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 200, + data: expect.objectContaining({ + result: expect.objectContaining({ + indexCode: 'UKX', + gbpIndex: 9116.68749114, + }), + }), + }), + }, + ]) + expect(mockResponseCache.write).toHaveBeenNthCalledWith(2, 'default_single_transport', [ + { + params: { instrument: 'Russell1000INDEX' }, + response: expect.objectContaining({ + statusCode: 200, + data: expect.objectContaining({ + result: expect.objectContaining({ + indexName: 'Russell 1000® Index', + close: 3547.4, + }), + }), + }), + }, + ]) + }) + + it('should handle errors gracefully in background handler', async () => { + mockSftpClient.connect.mockRejectedValue(new Error('Connection failed')) + + const mockContext: MockContext = { + adapterSettings: mockAdapterSettings, + } + + const entries: Array<{ instrument: string }> = [{ instrument: 'FTSE100INDEX' }] + + await transport.backgroundHandler( + mockContext as unknown as EndpointContext, + entries, + ) + + // Verify error was handled and cached + expect(mockResponseCache.write).toHaveBeenCalledWith('default_single_transport', [ + { + params: { instrument: 'FTSE100INDEX' }, + response: expect.objectContaining({ + statusCode: 502, + errorMessage: 'Failed to connect to SFTP server: Connection failed', + }), + }, + ]) + }) + }) + + describe('subscription configuration', () => { + it('should return correct subscription TTL from config', () => { + const ttl = transport.getSubscriptionTtlFromConfig(mockAdapterSettings) + expect(ttl).toBe(60000) + }) + + it('should return default TTL when BACKGROUND_EXECUTE_MS is not set', () => { + const settingsWithoutExecuteMs = makeStub('settingsWithoutExecuteMs', { + BACKGROUND_EXECUTE_MS: 0, // This will trigger the || 60000 fallback + } as unknown as BaseEndpointTypes['Settings']) + + const ttl = transport.getSubscriptionTtlFromConfig(settingsWithoutExecuteMs) + expect(ttl).toBe(60000) + }) + }) +})