feat(etls): add interreg 2014UK16RFOP005 - EUBFR-258

config.example.json

@@ -19,6 +19,7 @@
     "2014tc16rftn002",
     "2014tc16rftn003",
     "2014tc16rftn004",
+    "2014uk16rfop005",
     "bulgaria",
     "cordis",
     "devco",

docs/types/README.md

@@ -25,6 +25,7 @@ Here's a list of the transformations made in ETLs around the `Project` model.
 - [2014tc16rftn002 - XLS](./etls/2014tc16rftn002-xls.md)
 - [2014tc16rftn003 - XLS](./etls/2014tc16rftn003-xls.md)
 - [2014tc16rftn004 - XLS](./etls/2014tc16rftn004-xls.md)
+- [2014uk16rfop005 - XLS](./etls/2014uk16rfop005-xls.md)
 - [bulgaria - XLS](./etls/bulgaria-xls.md)
 - [CORDIS - CSV](./etls/cordis-csv.md)
 - [DEVCO - XLS](./etls/devco-xls.md)

docs/types/etls/2014uk16rfop005-xls.md

@@ -0,0 +1,173 @@
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+## 2014uk16rfop005XlsTransform
+
+Map fields for 2014uk16rfop005 producer, XLS file types
+
+Example input data: [stub][1]
+
+Transform function: [implementation details][2]
+
+### Parameters
+
+- `record` **[Object][3]** Piece of data to transform before going to harmonized storage.
+
+Returns **Project** JSON matching the type fields.
+
+### getBudget
+
+Preprocess `budget`.
+
+Input fields taken from the `record` are:
+
+- `EU funds awarded`
+- `Total Project cost`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **Budget**
+
+### getDescription
+
+Preprocess `description`.
+
+Input fields taken from the `record` are:
+
+- `Project Description`
+- `Sector`
+- `Union co-financing rate, as per Priority Axis`
+- `Welsh Government Targeted Match Funding, Yes / No’`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[String][4]**
+
+### getPriorities
+
+Preprocess `ec_priorities`.
+
+Input fields taken from the `record` are:
+
+- `Priority`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[Array][5]&lt;[String][4]>**
+
+### getProgramme
+
+Preprocess `programme_name`.
+
+Input fields taken from the `record` are:
+
+- `EU Programme`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[String][4]**
+
+### getProjectId
+
+Preprocess `project_id`.
+
+Uses `Case ID`, but if it's not present, the ID is generated based on `Project Title`.
+
+Input fields taken from the `record` are:
+
+- `Case ID`
+- `Project Title`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[String][4]**
+
+### getLocations
+
+Preprocess `project_locations`.
+
+Adds a default value of GB, because data is related to Wales.
+
+Input fields taken from the `record` are:
+
+- `Regional area(s)`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[Array][5]&lt;[Location][6]>**
+
+### getThemes
+
+Preprocess `themes`.
+
+Input fields taken from the `record` are:
+
+- `Category of intervention`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[Array][5]&lt;[String][4]>**
+
+### getThirdParties
+
+Preprocess `third_parties`.
+
+Input fields taken from the `record` are:
+
+- `Lead Organisation`
+- `Joint Sponsors`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[Array][5]&lt;ThirdParty>**
+
+### getTimeframe
+
+Preprocess `timeframe`.
+
+Input fields taken from the `record` are:
+
+- `Project start date`
+- `Project end date`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **Timeframe**
+
+### getTitle
+
+Preprocess `title`.
+
+Input fields taken from the `record` are:
+
+- `Project Title`
+
+#### Parameters
+
+- `record` **[Object][3]** The row received from parsed file
+
+Returns **[String][4]**
+
+[1]: https://github.com/ec-europa/eubfr-data-lake/blob/master/services/ingestion/etl/2014uk16rfop005/xls/test/stubs/record.json
+[2]: https://github.com/ec-europa/eubfr-data-lake/blob/master/services/ingestion/etl/2014uk16rfop005/xls/src/lib/transform.js
+[3]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object
+[4]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String
+[5]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array
+[6]: https://developer.mozilla.org/docs/Web/API/Location

scripts/documentation/docs-md.js

@@ -27,6 +27,7 @@ const transforms = [
   '2014tc16rftn002-xls',
   '2014tc16rftn003-xls',
   '2014tc16rftn004-xls',
+  '2014uk16rfop005-xls',
   'bulgaria-xls',
   'cordis-csv',
   'devco-xls',

services/ingestion/etl/2014uk16rfop005/xls/README.md

@@ -0,0 +1,22 @@
+# 2014uk16rfop005 XLS ETL mapping rules
+
+Model to compare with is available at: https://ec-europa.github.io/eubfr-data-lake/
+
+| Field                                              | Target            |
+| -------------------------------------------------- | ----------------- |
+| Project Title                                      | title             |
+| EU Programme                                       | programme_name    |
+| Priority                                           | ec_priorities     |
+| Project Description                                | description       |
+| Lead Organisation                                  | third_parties     |
+| Sector                                             | description       |
+| Joint Sponsors                                     | third_parties     |
+| Regional area(s)                                   | project_locations |
+| Category of intervention                           | themes            |
+| Project start date                                 | timeframe.from    |
+| Project end date                                   | timeframe.to      |
+| EU funds awarded                                   | budget.eu_contrib |
+| Total Project cost                                 | budget.total_cost |
+| Union co-financing rate, as per Priority Axis      | description       |
+| Welsh Government Targeted Match Funding, Yes / No’ | description       |
+| Case ID                                            | project_id        |

services/ingestion/etl/2014uk16rfop005/xls/babel.config.js

@@ -0,0 +1,29 @@
+module.exports = {
+  presets: [
+    '@babel/preset-flow',
+    [
+      '@babel/preset-env',
+      {
+        targets: {
+          node: '8.10',
+        },
+        modules: false,
+        loose: true,
+      },
+    ],
+  ],
+  env: {
+    test: {
+      presets: [
+        [
+          '@babel/preset-env',
+          {
+            targets: {
+              node: '8.10',
+            },
+          },
+        ],
+      ],
+    },
+  },
+};

services/ingestion/etl/2014uk16rfop005/xls/package.json

@@ -0,0 +1,32 @@
+{
+  "private": true,
+  "name": "@eubfr/ingestion-etl-2014uk16rfop005-xls",
+  "version": "0.7.0",
+  "scripts": {
+    "deploy": "sls deploy -v",
+    "test:unit": "jest --testPathPattern=unit"
+  },
+  "dependencies": {
+    "@eubfr/lib": "^0.7.0",
+    "@eubfr/logger-messenger": "^0.7.0",
+    "xlsx": "0.14.2"
+  },
+  "devDependencies": {
+    "@babel/core": "7.4.3",
+    "@babel/preset-env": "7.4.3",
+    "@babel/preset-flow": "7.0.0",
+    "@eubfr/types": "^0.7.0",
+    "aws-sdk": "2.434.0",
+    "babel-jest": "24.7.0",
+    "babel-loader": "8.0.5",
+    "jest": "24.7.0",
+    "serverless": "1.40.0",
+    "serverless-webpack": "5.2.0",
+    "webpack": "4.29.6"
+  },
+  "jest": {
+    "transform": {
+      "^.+\\.js$": "babel-jest"
+    }
+  }
+}