diff --git a/config.example.json b/config.example.json index ac57a6282..e674a5cdc 100644 --- a/config.example.json +++ b/config.example.json @@ -13,6 +13,7 @@ "2014tc16rfcb047", "2014tc16rfpc001", "2014tc16rftn002", + "2014uk16rfop003", "bulgaria", "cordis", "devco", diff --git a/docs/types/README.md b/docs/types/README.md index 1e3daea98..37e611be2 100644 --- a/docs/types/README.md +++ b/docs/types/README.md @@ -19,6 +19,7 @@ Here's a list of the transformations made in ETLs around the `Project` model. - [2014tc16rfcb047 - XLS](./etls/2014tc16rfcb047-xls.md) - [2014tc16rfpc001 - XLS](./etls/2014tc16rfpc001-xls.md) - [2014tc16rftn002 - XLS](./etls/2014tc16rftn002-xls.md) +- [2014uk16rfop003 - XLS](./etls/2014uk16rfop003-xls.md) - [bulgaria - XLS](./etls/bulgaria-xls.md) - [CORDIS - CSV](./etls/cordis-csv.md) - [DEVCO - XLS](./etls/devco-xls.md) diff --git a/docs/types/etls/2014uk16rfop003-xls.md b/docs/types/etls/2014uk16rfop003-xls.md new file mode 100644 index 000000000..685c9e953 --- /dev/null +++ b/docs/types/etls/2014uk16rfop003-xls.md @@ -0,0 +1,114 @@ + + +## 2014uk16rfop003XlsTransform + +Map fields for 2014uk16rfop003 producer, XLS file types + +Example input data: [stub][1] + +Transform function: [implementation details][2] + +### Parameters + +- `record` **[Object][3]** Piece of data to transform before going to harmonized storage. + +Returns **Project** JSON matching the type fields. + +### getBudget + +Preprocess `budget`. + +Input fields taken from the `record` are: + +- `Total Paid Amount` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **Budget** + +### getProjectId + +Preprocess `project_id`. + +Input fields taken from the `record` are: + +- `Ref. No` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[String][4]** + +### getLocations + +Preprocess `project_locations`. + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[Array][5]<[Location][6]>** + +### getThirdParties + +Preprocess `third_parties`. + +Input fields taken from the `record` are: + +- `Organisation Name` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[Array][5]<ThirdParty>** + +### formatDate + +Preprocess/format date. + +#### Parameters + +- `date` **[Date][7]** DateSupported formats:- `DD/MM/YYYY` + +Returns **[Date][7]** The date formatted into an ISO 8601 date format + +### getTimeframe + +Preprocess `timeframe`. + +Input fields taken from the `record` are: + +- `Operation Start Date` +- `Operation End Date` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **Timeframe** + +### getTitle + +Preprocess `title`. + +Input fields taken from the `record` are: + +- `Project Title` + +#### Parameters + +- `record` **[Object][3]** The row received from parsed file + +Returns **[String][4]** + +[1]: https://github.com/ec-europa/eubfr-data-lake/blob/master/services/ingestion/etl/2014uk16rfop003/xls/test/stubs/record.json +[2]: https://github.com/ec-europa/eubfr-data-lake/blob/master/services/ingestion/etl/2014uk16rfop003/xls/src/lib/transform.js +[3]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object +[4]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String +[5]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array +[6]: https://developer.mozilla.org/docs/Web/API/Location +[7]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date diff --git a/scripts/documentation/docs-md.js b/scripts/documentation/docs-md.js index fea1537ad..286afad79 100755 --- a/scripts/documentation/docs-md.js +++ b/scripts/documentation/docs-md.js @@ -21,6 +21,7 @@ const transforms = [ '2014tc16rfcb047-xls', '2014tc16rfpc001-xls', '2014tc16rftn002-xls', + '2014uk16rfop003-xls', 'bulgaria-xls', 'cordis-csv', 'devco-xls', diff --git a/services/ingestion/etl/2014uk16rfop003/xls/README.md b/services/ingestion/etl/2014uk16rfop003/xls/README.md new file mode 100644 index 000000000..8fc0608ad --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/README.md @@ -0,0 +1,14 @@ +# 2014uk16rfop003 XLS ETL mapping rules + +Model to compare with is available at: https://ec-europa.github.io/eubfr-data-lake/ + +| Field | Target | +| -------------------- | ----------------- | +| Ref. No | project_id | +| Organisation Name | third_parties | +| Project Title | title | +| Operation Start Date | timeframe.from | +| Operation End Date | timeframe.to | +| Total Grant Approved | | +| Total Paid Amount | budget.total_cost | +| Project Postcode | project_locations | diff --git a/services/ingestion/etl/2014uk16rfop003/xls/babel.config.js b/services/ingestion/etl/2014uk16rfop003/xls/babel.config.js new file mode 100644 index 000000000..0397ff2b1 --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/babel.config.js @@ -0,0 +1,29 @@ +module.exports = { + presets: [ + '@babel/preset-flow', + [ + '@babel/preset-env', + { + targets: { + node: '8.10', + }, + modules: false, + loose: true, + }, + ], + ], + env: { + test: { + presets: [ + [ + '@babel/preset-env', + { + targets: { + node: '8.10', + }, + }, + ], + ], + }, + }, +}; diff --git a/services/ingestion/etl/2014uk16rfop003/xls/package.json b/services/ingestion/etl/2014uk16rfop003/xls/package.json new file mode 100644 index 000000000..bab598ee3 --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/package.json @@ -0,0 +1,32 @@ +{ + "private": true, + "name": "@eubfr/ingestion-etl-2014uk16rfop003-xls", + "version": "0.7.0", + "scripts": { + "deploy": "sls deploy -v", + "test:unit": "jest --testPathPattern=unit" + }, + "dependencies": { + "@eubfr/lib": "^0.7.0", + "@eubfr/logger-messenger": "^0.7.0", + "xlsx": "0.14.2" + }, + "devDependencies": { + "@babel/core": "7.4.3", + "@babel/preset-env": "7.4.3", + "@babel/preset-flow": "7.0.0", + "@eubfr/types": "^0.7.0", + "aws-sdk": "2.434.0", + "babel-jest": "24.7.0", + "babel-loader": "8.0.5", + "jest": "24.7.0", + "serverless": "1.40.0", + "serverless-webpack": "5.2.0", + "webpack": "4.29.6" + }, + "jest": { + "transform": { + "^.+\\.js$": "babel-jest" + } + } +} diff --git a/services/ingestion/etl/2014uk16rfop003/xls/serverless.yml b/services/ingestion/etl/2014uk16rfop003/xls/serverless.yml new file mode 100644 index 000000000..6a7def84d --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/serverless.yml @@ -0,0 +1,123 @@ +service: ingestion-etl-2014uk16rfop003-xls + +plugins: + - serverless-webpack + +custom: + webpack: + webpackConfig: ./webpack.config.js + includeModules: + forceExclude: + - aws-sdk + packager: yarn + eubfrEnvironment: ${opt:eubfr_env, file(../../../../../config.json):eubfr_env, env:EUBFR_ENV, 'dev'} + bucketName: ${file(../../../../../resources/harmonized-storage/serverless.yml):custom.bucketName} + +package: + individually: true + +provider: + name: aws + runtime: nodejs8.10 + timeout: 60 + stage: ${opt:stage, file(../../../../../config.json):stage, env:EUBFR_STAGE, 'dev'} + region: ${opt:region, file(../../../../../config.json):region, env:EUBFR_AWS_REGION, 'eu-central-1'} + deploymentBucket: + name: eubfr-${self:custom.eubfrEnvironment}-deploy + stackTags: + ENV: ${self:custom.eubfrEnvironment} + iamRoleStatements: + - Effect: 'Allow' + Action: + - 's3:PutObject' + Resource: + Fn::Join: + - '' + - - 'arn:aws:s3:::' + - ${self:custom.bucketName} + - '/*' + # Allow queueing messages to the DLQ https://docs.aws.amazon.com/lambda/latest/dg/dlq.html + - Effect: 'Allow' + Action: + - sqs:SendMessage + Resource: '*' + +functions: + parseXls: + handler: src/events/onParseXLS.handler + name: ${self:provider.stage}-${self:service}-parseXls + memorySize: 1024 + environment: + BUCKET: ${self:custom.bucketName} + REGION: ${self:provider.region} + STAGE: ${self:provider.stage} + events: + - sns: + arn: + Fn::Join: + - '' + - - 'arn:aws:sns:' + - Ref: 'AWS::Region' + - ':' + - Ref: 'AWS::AccountId' + - ':${self:provider.stage}-etl-2014uk16rfop003-xls' + topicName: ${self:provider.stage}-etl-2014uk16rfop003-xls + - sns: + arn: + Fn::Join: + - '' + - - 'arn:aws:sns:' + - Ref: 'AWS::Region' + - ':' + - Ref: 'AWS::AccountId' + - ':${self:provider.stage}-etl-2014uk16rfop003-xlsx' + topicName: ${self:provider.stage}-etl-2014uk16rfop003-xlsx + +resources: + Resources: + ParseXlsLambdaFunction: + Type: 'AWS::Lambda::Function' + Properties: + DeadLetterConfig: + TargetArn: + Fn::ImportValue: ${self:provider.stage}:ingestion-dead-letter-queue:LambdaFailureQueue + SNSTopic2014uk16rfop003XLS: + Type: AWS::SNS::Topic + Properties: + TopicName: ${self:provider.stage}-etl-2014uk16rfop003-xls + DisplayName: 2014uk16rfop003 XLS ETL + SNSTopic2014uk16rfop003XLSX: + Type: AWS::SNS::Topic + Properties: + TopicName: ${self:provider.stage}-etl-2014uk16rfop003-xlsx + DisplayName: 2014uk16rfop003 XLSX ETL + SNSTopic2014uk16rfop003XLSPolicy: + Type: AWS::SNS::TopicPolicy + Properties: + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: Allow-IngestionManager-Publish + Action: + - sns:Publish + Effect: Allow + Resource: + Fn::Join: + - '' + - - 'arn:aws:sns:' + - Ref: 'AWS::Region' + - ':' + - Ref: 'AWS::AccountId' + - ':${self:provider.stage}-etl-2014uk16rfop003-*' + Principal: + AWS: + Fn::Join: + - '' + - - 'arn:aws:sts::' + - Ref: 'AWS::AccountId' + - ':assumed-role/ingestion-manager-${self:provider.stage}-' + - Ref: 'AWS::Region' + - '-lambdaRole/${self:provider.stage}-ingestion-manager-onObjectCreated' + Topics: + - Ref: SNSTopic2014uk16rfop003XLS + - Ref: SNSTopic2014uk16rfop003XLSX diff --git a/services/ingestion/etl/2014uk16rfop003/xls/src/events/onParseXLS.js b/services/ingestion/etl/2014uk16rfop003/xls/src/events/onParseXLS.js new file mode 100644 index 000000000..840eb0af1 --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/src/events/onParseXLS.js @@ -0,0 +1,108 @@ +import AWS from 'aws-sdk'; // eslint-disable-line import/no-extraneous-dependencies +import XLSX from 'xlsx'; + +// ETL utilities. +import ensureExtensions from '@eubfr/lib/etl/ensureExtensions'; +import extractMessage from '@eubfr/lib/etl/extractMessage'; +import handleError from '@eubfr/lib/etl/handleError'; + +import MessengerFactory from '@eubfr/logger-messenger/src/lib/MessengerFactory'; +import { STATUS } from '@eubfr/logger-messenger/src/lib/status'; + +import transformRecord from '../lib/transform'; + +export const handler = async (event, context) => { + const { BUCKET, REGION, STAGE } = process.env; + + if (!BUCKET || !REGION || !STAGE) { + throw new Error( + 'BUCKET, REGION and STAGE environment variables are required!' + ); + } + + try { + const snsMessage = extractMessage(event); + const { key } = snsMessage.object; + + if (!ensureExtensions({ file: key, extensions: ['.xls', '.xlsx'] })) { + throw new Error('XLS or XLSX file expected for this ETL.'); + } + + const messenger = MessengerFactory.Create({ context }); + const s3 = new AWS.S3(); + + await messenger.send({ + message: { + computed_key: key, + status_message: 'Start parsing XLS...', + status_code: STATUS.PARSING, + }, + to: ['logs'], + }); + + // Get file + const readStream = s3 + .getObject({ Bucket: snsMessage.bucket.name, Key: key }) + .createReadStream(); + + return new Promise((resolve, reject) => { + // Put data in buffer + const buffers = []; + + readStream.on('data', data => { + buffers.push(data); + }); + + readStream.on('error', async e => + handleError( + { messenger, key, statusCode: STATUS.ERROR }, + { error: e, callback: reject } + ) + ); + + // Manage data + readStream.on('end', async () => { + let dataString = ''; + + // Parse file + const buffer = Buffer.concat(buffers); + const workbook = XLSX.read(buffer); + const sheetNameList = workbook.SheetNames; + const parsedRows = XLSX.utils.sheet_to_json( + workbook.Sheets[sheetNameList[0]] + ); + + parsedRows.forEach(record => { + const data = transformRecord(record); + dataString += `${JSON.stringify(data)}\n`; + }); + + // Load data + const params = { + Bucket: BUCKET, + Key: `${key}.ndjson`, + Body: dataString, + ContentType: 'application/x-ndjson', + }; + + await s3.upload(params).promise(); + + await messenger.send({ + message: { + computed_key: key, + status_message: + 'XLS parsed successfully. Results will be uploaded to ElasticSearch soon...', + status_code: STATUS.PARSED, + }, + to: ['logs'], + }); + + return resolve('XLS parsed successfully'); + }); + }); + } catch (e) { + throw e; + } +}; + +export default handler; diff --git a/services/ingestion/etl/2014uk16rfop003/xls/src/lib/transform.js b/services/ingestion/etl/2014uk16rfop003/xls/src/lib/transform.js new file mode 100644 index 000000000..f6682aa75 --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/src/lib/transform.js @@ -0,0 +1,206 @@ +// @flow + +import type { Project } from '@eubfr/types'; +import sanitizeBudgetItem from '@eubfr/lib/budget/budgetFormatter'; + +/** + * Preprocess `budget`. + * + * Input fields taken from the `record` are: + * - `Total Paid Amount` + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Object} record The row received from parsed file + * @returns {Budget} + */ + +const getBudget = record => ({ + total_cost: sanitizeBudgetItem({ + value: record['Total Paid Amount'], + currency: 'GBP', + raw: record['Total Paid Amount'], + }), + eu_contrib: sanitizeBudgetItem(), + private_fund: sanitizeBudgetItem(), + public_fund: sanitizeBudgetItem(), + other_contrib: sanitizeBudgetItem(), + funding_area: [], + mmf_heading: '', +}); + +/** + * Preprocess `project_id`. + * + * Input fields taken from the `record` are: + * - `Ref. No` + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Object} record The row received from parsed file + * @returns {String} + */ + +const getProjectId = record => record['Ref. No'] || ''; + +/** + * Preprocess `project_locations`. + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Object} record The row received from parsed file + * @returns {Array} + */ + +const getLocations = record => [ + { + address: '', + centroid: null, + country_code: 'GB', // Takes into account lib/getCountryCode.js rules directly. + location: null, + nuts: [], + postal_code: record['Project Postcode'] + ? record['Project Postcode'].trim() + : '', + region: '', + town: '', + }, +]; + +/** + * Preprocess `third_parties`. + * + * Input fields taken from the `record` are: + * - `Organisation Name` + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Object} record The row received from parsed file + * @returns {Array} + */ + +const getThirdParties = record => + record['Organisation Name'] + ? [ + { + address: '', + country: 'GB', + email: '', + name: record['Organisation Name'] + ? record['Organisation Name'].trim() + : '', + phone: '', + region: '', + role: 'Lead organisation', + type: '', + website: '', + }, + ] + : []; + +/** + * Preprocess/format date. + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Date} date Date + * + * Supported formats: + * + * - `DD/MM/YYYY` + * + * @returns {Date} The date formatted into an ISO 8601 date format + * + */ +const formatDate = date => { + if (!date || typeof date !== 'string') return null; + + const d = date.split(/\//); + if (d.length !== 3) return null; + + const [day, month, year] = d; + + if (!day || !month || !year) return null; + + try { + return new Date(Date.UTC(year, month - 1, day)).toISOString(); + } catch (e) { + return null; + } +}; + +/** + * Preprocess `timeframe`. + * + * Input fields taken from the `record` are: + * - `Operation Start Date` + * - `Operation End Date` + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Object} record The row received from parsed file + * @returns {Timeframe} + */ + +const getTimeframe = record => { + const from = formatDate(record['Operation Start Date']); + const to = formatDate(record['Operation End Date']); + + return { + from, + from_precision: 'day', + to, + to_precision: 'day', + }; +}; + +/** + * Preprocess `title`. + * + * Input fields taken from the `record` are: + * - `Project Title` + * + * @memberof 2014uk16rfop003XlsTransform + * @param {Object} record The row received from parsed file + * @returns {String} + */ + +const getTitle = record => + record['Project Title'] ? record['Project Title'].trim() : ''; + +/** + * Map fields for 2014uk16rfop003 producer, XLS file types + * + * Example input data: {@link https://github.com/ec-europa/eubfr-data-lake/blob/master/services/ingestion/etl/2014uk16rfop003/xls/test/stubs/record.json|stub} + * + * Transform function: {@link https://github.com/ec-europa/eubfr-data-lake/blob/master/services/ingestion/etl/2014uk16rfop003/xls/src/lib/transform.js|implementation details} + * @name 2014uk16rfop003XlsTransform + * @param {Object} record Piece of data to transform before going to harmonized storage. + * @returns {Project} JSON matching the type fields. + */ +export default (record: Object): Project | null => { + if (!record) return null; + + // Map the fields + return { + action: '', + budget: getBudget(record), + call_year: '', + description: '', + ec_priorities: [], + media: [], + programme_name: '', + project_id: getProjectId(record), + project_locations: getLocations(record), + project_website: '', + complete: false, + related_links: [], + reporting_organisation: 'Member states', + results: { + available: '', + result: '', + }, + status: '', + sub_programme_name: '', + success_story: '', + themes: [], + third_parties: getThirdParties(record), + timeframe: getTimeframe(record), + title: getTitle(record), + type: [], + }; +}; diff --git a/services/ingestion/etl/2014uk16rfop003/xls/test/stubs/record.json b/services/ingestion/etl/2014uk16rfop003/xls/test/stubs/record.json new file mode 100644 index 000000000..4ca8e247c --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/test/stubs/record.json @@ -0,0 +1,10 @@ +{ + "Ref. No": "200048", + "Organisation Name": "ERDF Managing Authority", + "Project Title": "MA Staff Salaries", + "Operation Start Date": "29/06/2015", + "Operation End Date": "31/03/2017", + "Total Grant Approved": "£937,416.80", + "Total Paid Amount": "£905,031.51", + "Project Postcode": "BT2 8FD" +} diff --git a/services/ingestion/etl/2014uk16rfop003/xls/test/unit/events/onParseXLS.spec.js b/services/ingestion/etl/2014uk16rfop003/xls/test/unit/events/onParseXLS.spec.js new file mode 100644 index 000000000..c4294a537 --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/test/unit/events/onParseXLS.spec.js @@ -0,0 +1,20 @@ +/** + * @jest-environment node + */ + +import onParseXLS from '../../../src/events/onParseXLS'; + +describe(`Function onParseXLS in "@eubfr/ingestion-etl-2014uk16rfop003-xls"`, () => { + test('The function requires BUCKET, REGION and STAGE environment variables', async () => { + const event = {}; + const context = {}; + + try { + await onParseXLS(event, context); + } catch (error) { + expect(error.message).toEqual( + 'BUCKET, REGION and STAGE environment variables are required!' + ); + } + }); +}); diff --git a/services/ingestion/etl/2014uk16rfop003/xls/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/2014uk16rfop003/xls/test/unit/lib/__snapshots__/transform.spec.js.snap new file mode 100644 index 000000000..19400a74c --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/test/unit/lib/__snapshots__/transform.spec.js.snap @@ -0,0 +1,87 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`XLS transformer for 2014uk16rfop003 Produces correct JSON output structure 1`] = ` +Object { + "action": "", + "budget": Object { + "eu_contrib": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "funding_area": Array [], + "mmf_heading": "", + "other_contrib": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "private_fund": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "public_fund": Object { + "currency": "", + "raw": "", + "value": 0, + }, + "total_cost": Object { + "currency": "GBP", + "raw": "£905,031.51", + "value": 905031.51, + }, + }, + "call_year": "", + "complete": false, + "description": "", + "ec_priorities": Array [], + "media": Array [], + "programme_name": "", + "project_id": "200048", + "project_locations": Array [ + Object { + "address": "", + "centroid": null, + "country_code": "GB", + "location": null, + "nuts": Array [], + "postal_code": "BT2 8FD", + "region": "", + "town": "", + }, + ], + "project_website": "", + "related_links": Array [], + "reporting_organisation": "Member states", + "results": Object { + "available": "", + "result": "", + }, + "status": "", + "sub_programme_name": "", + "success_story": "", + "themes": Array [], + "third_parties": Array [ + Object { + "address": "", + "country": "GB", + "email": "", + "name": "ERDF Managing Authority", + "phone": "", + "region": "", + "role": "Lead organisation", + "type": "", + "website": "", + }, + ], + "timeframe": Object { + "from": "2015-06-29T00:00:00.000Z", + "from_precision": "day", + "to": "2017-03-31T00:00:00.000Z", + "to_precision": "day", + }, + "title": "MA Staff Salaries", + "type": Array [], +} +`; diff --git a/services/ingestion/etl/2014uk16rfop003/xls/test/unit/lib/transform.spec.js b/services/ingestion/etl/2014uk16rfop003/xls/test/unit/lib/transform.spec.js new file mode 100644 index 000000000..62576bf4a --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/test/unit/lib/transform.spec.js @@ -0,0 +1,22 @@ +/** + * @jest-environment node + */ + +import mapper from '../../../src/lib/transform'; +import testRecord from '../../stubs/record.json'; + +describe('XLS transformer for 2014uk16rfop003', () => { + let results = {}; + + beforeAll(() => { + results = mapper(testRecord); + }); + + test('Returns null when record is not provided', () => { + expect(mapper()).toBe(null); + }); + + test('Produces correct JSON output structure', () => { + expect(results).toMatchSnapshot(); + }); +}); diff --git a/services/ingestion/etl/2014uk16rfop003/xls/webpack.config.js b/services/ingestion/etl/2014uk16rfop003/xls/webpack.config.js new file mode 100644 index 000000000..30fd8ced7 --- /dev/null +++ b/services/ingestion/etl/2014uk16rfop003/xls/webpack.config.js @@ -0,0 +1,32 @@ +const slsw = require('serverless-webpack'); +const path = require('path'); + +module.exports = { + entry: slsw.lib.entries, + target: 'node', + mode: slsw.lib.webpack.isLocal ? 'development' : 'production', + optimization: { + minimize: process.env.EUBFR_ENV && process.env.EUBFR_ENV === 'prod', + }, + devtool: 'nosources-source-map', + externals: [{ 'aws-sdk': true }], + module: { + rules: [ + { + test: /\.js$/, + use: [ + { + loader: 'babel-loader', + }, + ], + include: __dirname, + exclude: /node_modules/, + }, + ], + }, + output: { + libraryTarget: 'commonjs2', + path: path.join(__dirname, '.webpack'), + filename: '[name].js', + }, +}; diff --git a/tools/eubfr-cli/lib/getServices.js b/tools/eubfr-cli/lib/getServices.js index a49b7899f..bc79fb433 100644 --- a/tools/eubfr-cli/lib/getServices.js +++ b/tools/eubfr-cli/lib/getServices.js @@ -18,6 +18,7 @@ const allServices = [ { service: 'ingestion-etl-2014tc16rfcb047-xls', exportEnv: false }, { service: 'ingestion-etl-2014tc16rfpc001-xls', exportEnv: false }, { service: 'ingestion-etl-2014tc16rftn002-xls', exportEnv: false }, + { service: 'ingestion-etl-2014uk16rfop003-xls', exportEnv: false }, { service: 'ingestion-etl-bulgaria-xls', exportEnv: false }, { service: 'ingestion-etl-cordis-csv', exportEnv: false }, { service: 'ingestion-etl-devco-xls', exportEnv: false },