OPDATA-3889: FTSE sFTP EA

.changeset/warm-hornets-compete.md

@@ -0,0 +1,5 @@
+---
+'@chainlink/ftse-sftp-adapter': major
+---
+
+Adding Downloading and parsing logic for russell and ftse csv files from ftse sftp server

packages/scripts/src/generate-readme/readmeBlacklist.json

@@ -64,7 +64,6 @@
     "xsushi-price",
     "coinbase-prime",
     "harris-and-trotter",
-    "nav-consulting",
-    "ftse-sftp"
+    "nav-consulting"
   ]
 }

packages/sources/ftse-sftp/src/config/index.ts

@@ -24,6 +24,11 @@ export const config = new AdapterConfig({
     sensitive: true,
     required: true,
   },
+  SFTP_READY_TIMEOUT_MS: {
+    description: 'How long (in milliseconds) to wait for the SSH handshake to complete',
+    type: 'number',
+    default: 30000,
+  },
   BACKGROUND_EXECUTE_MS: {
     description:
       'The amount of time the background execute should sleep before performing the next request',

packages/sources/ftse-sftp/src/endpoint/index.ts

@@ -1,3 +1 @@
-import * as sftp from './sftp'
-
-export { sftp }
+export { endpoint as sftp } from './sftp'

packages/sources/ftse-sftp/src/endpoint/sftp.ts

@@ -11,6 +11,7 @@ export const inputParameters = new InputParameters(
       required: true,
       type: 'string',
       description: 'Abstract identifier of the index to fetch the data for',
+      options: ['FTSE100INDEX', 'Russell1000INDEX', 'Russell2000INDEX', 'Russell3000INDEX'],
     },
   },
   [
@@ -29,7 +30,7 @@ export const inputParameters = new InputParameters(
   ],
 )
 
-export type TInputParameters = typeof inputParameters.definition
+export type Instrument = (typeof inputParameters.validated)['instrument']
 
 /**
  * Union type for all possible response data structures
@@ -41,14 +42,15 @@ export type BaseEndpointTypes = {
   Response: {
     Result: number
     Data: {
+      filename: string
       result: IndexResponseData
     }
   }
   Settings: typeof config.settings
 }
 
 export const endpoint = new AdapterEndpoint({
-  name: 'ftse_sftp',
+  name: 'sftp',
   transport: sftpTransport,
   inputParameters,
 })

packages/sources/ftse-sftp/src/index.ts

@@ -1 +1,13 @@
-export * from './parsing'
+import { expose, ServerInstance } from '@chainlink/external-adapter-framework'
+import { Adapter } from '@chainlink/external-adapter-framework/adapter'
+import { config } from './config'
+import { sftp } from './endpoint'
+
+export const adapter = new Adapter({
+  defaultEndpoint: sftp.name,
+  name: 'FTSE_SFTP',
+  config,
+  endpoints: [sftp],
+})
+
+export const server = (): Promise<ServerInstance | undefined> => expose(adapter)

packages/sources/ftse-sftp/src/transport/constants.ts

@@ -0,0 +1,23 @@
+import { Instrument } from '../endpoint/sftp'
+
+export const FTSE100INDEX: Instrument = 'FTSE100INDEX'
+export const RUSSELL_1000_INDEX: Instrument = 'Russell1000INDEX'
+export const RUSSELL_2000_INDEX: Instrument = 'Russell2000INDEX'
+export const RUSSELL_3000_INDEX: Instrument = 'Russell3000INDEX'
+
+export const instrumentToDirectoryMap: Record<Instrument, string> = {
+  [FTSE100INDEX]: '/data/valuation/uk_all_share/',
+  [RUSSELL_1000_INDEX]:
+    '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/',
+  [RUSSELL_2000_INDEX]:
+    '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/',
+  [RUSSELL_3000_INDEX]:
+    '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/',
+}
+
+export const instrumentToFileRegexMap: Record<Instrument, RegExp> = {
+  [FTSE100INDEX]: /^ukallv\d{4}\.csv$/,
+  [RUSSELL_1000_INDEX]: /^daily_values_russell_\d{6}\.CSV$/,
+  [RUSSELL_2000_INDEX]: /^daily_values_russell_\d{6}\.CSV$/,
+  [RUSSELL_3000_INDEX]: /^daily_values_russell_\d{6}\.CSV$/,
+}

packages/sources/ftse-sftp/src/transport/sftp.ts

@@ -1,18 +1,23 @@
 import { EndpointContext } from '@chainlink/external-adapter-framework/adapter'
 import { TransportDependencies } from '@chainlink/external-adapter-framework/transports'
 import { SubscriptionTransport } from '@chainlink/external-adapter-framework/transports/abstract/subscription'
-import { sleep } from '@chainlink/external-adapter-framework/util'
-import SftpClient from 'ssh2-sftp-client'
-import { BaseEndpointTypes } from '../endpoint/sftp'
+import { AdapterResponse, makeLogger, sleep } from '@chainlink/external-adapter-framework/util'
+import { AdapterInputError } from '@chainlink/external-adapter-framework/validation/error'
+import { ConnectOptions } from 'ssh2-sftp-client'
+import { BaseEndpointTypes, IndexResponseData, inputParameters, Instrument } from '../endpoint/sftp'
+import { CSVParserFactory } from '../parsing/factory'
+import { instrumentToDirectoryMap, instrumentToFileRegexMap } from './constants'
+import { getFileContentsFromFileRegex } from './utils'
+
+const logger = makeLogger('FTSE SFTP Adapter')
+
+type RequestParams = typeof inputParameters.validated
 
 export class SftpTransport extends SubscriptionTransport<BaseEndpointTypes> {
   config!: BaseEndpointTypes['Settings']
-  endpointName!: string
-  sftpClient: SftpClient
 
   constructor() {
     super()
-    this.sftpClient = new SftpClient()
   }
 
   async initialize(
@@ -23,13 +28,102 @@ export class SftpTransport extends SubscriptionTransport<BaseEndpointTypes> {
   ): Promise<void> {
     await super.initialize(dependencies, adapterSettings, endpointName, transportName)
     this.config = adapterSettings
-    this.endpointName = endpointName
   }
 
-  async backgroundHandler(context: EndpointContext<BaseEndpointTypes>): Promise<void> {
+  async backgroundHandler(
+    context: EndpointContext<BaseEndpointTypes>,
+    entries: RequestParams[],
+  ): Promise<void> {
+    await Promise.all(entries.map(async (param) => this.handleRequest(param)))
     await sleep(context.adapterSettings.BACKGROUND_EXECUTE_MS)
   }
 
+  async handleRequest(param: RequestParams) {
+    let response: AdapterResponse<BaseEndpointTypes['Response']>
+    try {
+      response = await this._handleRequest(param)
+    } catch (e) {
+      const errorMessage = e instanceof Error ? e.message : 'Unknown error occurred'
+      response = {
+        statusCode: 502,
+        errorMessage,
+        timestamps: {
+          providerDataRequestedUnixMs: 0,
+          providerDataReceivedUnixMs: 0,
+          providerIndicatedTimeUnixMs: undefined,
+        },
+      }
+    }
+
+    await this.responseCache.write(this.name, [{ params: param, response }])
+  }
+
+  async _handleRequest(
+    param: RequestParams,
+  ): Promise<AdapterResponse<BaseEndpointTypes['Response']>> {
+    const providerDataRequestedUnixMs = Date.now()
+
+    const { filename, result, parsedData } = await this.tryDownloadAndParseFile(param.instrument)
+
+    logger.debug(`Successfully processed data for instrument: ${param.instrument}`)
+    return {
+      data: {
+        filename,
+        result: parsedData,
+      },
+      statusCode: 200,
+      result,
+      timestamps: {
+        providerDataRequestedUnixMs,
+        providerDataReceivedUnixMs: Date.now(),
+        providerIndicatedTimeUnixMs: undefined,
+      },
+    }
+  }
+
+  private async tryDownloadAndParseFile(instrument: Instrument): Promise<{
+    filename: string
+    result: number
+    parsedData: IndexResponseData
+  }> {
+    const connectOptions: ConnectOptions = {
+      host: this.config.SFTP_HOST,
+      port: this.config.SFTP_PORT,
+      username: this.config.SFTP_USERNAME,
+      password: this.config.SFTP_PASSWORD,
+      readyTimeout: this.config.SFTP_READY_TIMEOUT_MS,
+    }
+
+    const directory = instrumentToDirectoryMap[instrument]
+    const filenameRegex = instrumentToFileRegexMap[instrument]
+
+    const { filename, fileContent } = await getFileContentsFromFileRegex({
+      connectOptions,
+      directory,
+      filenameRegex,
+    })
+
+    // we need latin1 here because the file contains special characters like "®"
+    const csvContent = fileContent.toString('latin1')
+
+    const parser = CSVParserFactory.detectParserByInstrument(instrument)
+
+    if (!parser) {
+      throw new AdapterInputError({
+        statusCode: 500,
+        message: `Parser initialization failed for instrument: ${instrument}`,
+      })
+    }
+
+    const { result, parsedData } = await parser.parse(csvContent)
+
+    return {
+      filename,
+      result,
+      parsedData: parsedData as IndexResponseData,
+    }
+  }
+
   getSubscriptionTtlFromConfig(adapterSettings: BaseEndpointTypes['Settings']): number {
     return adapterSettings.BACKGROUND_EXECUTE_MS || 60000
   }

packages/sources/ftse-sftp/test/fixtures/index.ts

@@ -1,5 +1,6 @@
 import fs from 'fs'
 import path from 'path'
+import { FileInfo } from 'ssh2-sftp-client'
 
 // Helper function to read fixture files
 export function readFixtureFile(filename: string): string {
@@ -24,23 +25,26 @@ export const expectedRussellData = {
   close: 3547.4,
 }
 
-// Test data generation helpers
-export const createFTSETestData = (dataRows: string[]): string => {
-  const header = `02/09/2025 (C) FTSE International Limited 2025. All Rights Reserved
-FTSE UK All-Share Indices Valuation Service
+export const ftseFilename = 'ukallv0209.csv'
+export const russellFilename = 'daily_values_russell_250827.CSV'
 
-Index Code,Index/Sector Name,Number of Constituents,Index Base Currency,USD Index,GBP Index,EUR Index,JPY Index,AUD Index,CNY Index,HKD Index,CAD Index,LOC Index,Base Currency (GBP) Index,USD TRI,GBP TRI,EUR TRI,JPY TRI,AUD TRI,CNY TRI,HKD TRI,CAD TRI,LOC TRI,Base Currency (GBP) TRI,Mkt Cap (USD),Mkt Cap (GBP),Mkt Cap (EUR),Mkt Cap (JPY),Mkt Cap (AUD),Mkt Cap (CNY),Mkt Cap (HKD),Mkt Cap (CAD),Mkt Cap (LOC),Mkt Cap Base Currency (GBP),XD Adjustment (YTD),Dividend Yield`
+export const ftseDirectory = '/data/valuation/uk_all_share/'
+export const russellDirectory =
+  '/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/'
 
-  return header + '\n' + dataRows.join('\n')
+export const fileContents: Record<string, string> = {
+  [path.join(ftseDirectory, ftseFilename)]: ftseCsvFixture,
+  [path.join(russellDirectory, russellFilename)]: russellCsvFixture,
 }
 
-export const createRussellTestData = (dataRows: string[]): string => {
-  const header = `"Daily Values",,,,,,,,,,,,,,
-,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,
-"As of August 27, 2025",,,,,,,,"Last 5 Trading Days",,,,"1 Year Ending",,
-,,,,,,,,"Closing Values",,,,"Closing Values",,
-,"Open","High","Low","Close","Net Chg","% Chg","High","Low","Net Chg","% Chg","High","Low","Net Chg","% Chg"`
-
-  return header + '\n' + dataRows.join('\n')
-}
+export const directoryListings = {
+  [ftseDirectory]: [
+    'vall_icb2302.csv',
+    'vall1809.csv',
+    'valllst.csv',
+    ftseFilename,
+    'ukallvlst.csv',
+    'vall_icb2302_v1.csv',
+  ].map((name) => ({ name })),
+  [russellDirectory]: ['history', russellFilename].map((name) => ({ name })),
+} as unknown as Record<string, FileInfo[]>

packages/sources/ftse-sftp/test/integration/__snapshots__/adapter.test.ts.snap

@@ -0,0 +1,87 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`execute sftp endpoint should return error for unknown instrument 1`] = `
+{
+  "error": {
+    "message": "[Param: instrument] input is not one of valid options (FTSE100INDEX,Russell1000INDEX,Russell2000INDEX,Russell3000INDEX)",
+    "name": "AdapterError",
+  },
+  "status": "errored",
+  "statusCode": 400,
+}
+`;
+
+exports[`execute sftp endpoint should return success for FTSE100INDEX 1`] = `
+{
+  "data": {
+    "filename": "ukallv0209.csv",
+    "result": {
+      "gbpIndex": 9116.68749114,
+      "indexBaseCurrency": "GBP",
+      "indexCode": "UKX",
+      "indexSectorName": "FTSE 100 Index",
+      "numberOfConstituents": 100,
+    },
+  },
+  "result": 9116.68749114,
+  "statusCode": 200,
+  "timestamps": {
+    "providerDataReceivedUnixMs": 978347471111,
+    "providerDataRequestedUnixMs": 978347471111,
+  },
+}
+`;
+
+exports[`execute sftp endpoint should return success for Russell1000INDEX 1`] = `
+{
+  "data": {
+    "filename": "daily_values_russell_250827.CSV",
+    "result": {
+      "close": 3547.4,
+      "indexName": "Russell 1000® Index",
+    },
+  },
+  "result": 3547.4,
+  "statusCode": 200,
+  "timestamps": {
+    "providerDataReceivedUnixMs": 978347471111,
+    "providerDataRequestedUnixMs": 978347471111,
+  },
+}
+`;
+
+exports[`execute sftp endpoint should return success for Russell2000INDEX 1`] = `
+{
+  "data": {
+    "filename": "daily_values_russell_250827.CSV",
+    "result": {
+      "close": 2373.8,
+      "indexName": "Russell 2000® Index",
+    },
+  },
+  "result": 2373.8,
+  "statusCode": 200,
+  "timestamps": {
+    "providerDataReceivedUnixMs": 978347471111,
+    "providerDataRequestedUnixMs": 978347471111,
+  },
+}
+`;
+
+exports[`execute sftp endpoint should return success for Russell3000INDEX 1`] = `
+{
+  "data": {
+    "filename": "daily_values_russell_250827.CSV",
+    "result": {
+      "close": 3690.93,
+      "indexName": "Russell 3000® Index",
+    },
+  },
+  "result": 3690.93,
+  "statusCode": 200,
+  "timestamps": {
+    "providerDataReceivedUnixMs": 978347471111,
+    "providerDataRequestedUnixMs": 978347471111,
+  },
+}
+`;