feat: add taxonomy transform command [GROOT-1496] (#2797)

* feat: add taxonomy transform command [GROOT-1506] WIP

* fix: type changes

* feat: introduce class with setters for Concept

* feat: add support for concept schemes, add example

* feat: clean up

* feat: add other csv example

* feat: add integration tests

* fix: update snapshot

* fix: small cleean up

* fix: remove space id from command requirements

* fix: fix column references in example test

* Update lib/cmds/organization_cmds/utils/taxonomy.ts

Co-authored-by: Jared Jolton <[email protected]>

* feat: update command params, add tests

---------

Co-authored-by: Jared Jolton <[email protected]>

Author: mayakarabula

.gitignore

@@ -192,3 +192,5 @@ contentful-import-error-log-*.json
 
 .idea
 reports
+
+data/*

docs/organization/taxonomy-transform/examples/example-data.csv

@@ -0,0 +1,30 @@
+animalia,animalia,A taxonomy focused on the categorization of various animals and their classifications.,,,,,,,,,,
+mammals,,,Mammals,mammals,,,,,,,,
+primates,,,,,Primates,primates,,,,,,
+felidae,,,,,Felidae (Cats),felidae,,,,,,
+canidae,,,,,Canidae (Dogs),canidae,,,,,,
+rodentia,,,,,Rodentia (Rodents),rodentia,,,,,,
+aves,,,Birds,aves,,,,,,,,
+passeriformes,,,,,Passeriformes (Perching Birds),passeriformes,,,,,,
+raptors,,,,,Raptors (Birds of Prey),raptors,,,,,,
+waterfowl,,,,,Waterfowl,waterfowl,,,,,,
+reptilia,,,Reptiles,reptilia,,,,,,,,
+squamata,,,,,Squamata (Lizards and Snakes),squamata,,,,,,
+testudines,,,,,Testudines (Turtles),testudines,,,,,,
+crocodylia,,,,,Crocodylia (Crocodiles),crocodylia,,,,,,
+amphibia,,,Amphibians,amphibia,,,,,,,,
+anura,,,,,Anura (Frogs and Toads),anura,,,,,,
+caudata,,,,,Caudata (Salamanders),caudata,,,,,,
+gymnophiona,,,,,Gymnophiona (Caecilians),gymnophiona,,,,,,
+pisces,,,Fish,fish,,,,,,,,
+chondrichthyes,,,,,Chondrichthyes (Cartilaginous Fish),chondrichthyes,,,,,,
+osteichthyes,,,,,Osteichthyes (Bony Fish),osteichthyes,,,,,,
+insecta,,,Insects,insecta,,,,,,,,
+lepidoptera,,,,,Lepidoptera (Butterflies and Moths),lepidoptera,,,,,,
+coleoptera,,,,,Coleoptera (Beetles),coleoptera,,,,,,
+diptera,,,,,Diptera (Flies),diptera,,,,,,
+hymenoptera,,,,,Hymenoptera (Bees Wasps Ants),hymenoptera,,,,,,
+arachnida,,,Arachnids,arachnida,,,,,,,,
+araneae,,,,,Araneae (Spiders),araneae,,,,,,
+scorpiones,,,,,Scorpiones (Scorpions),scorpiones,,,,,,
+opiliones,,,,,Opiliones (Harvestmen),opiliones,,,,,,

docs/organization/taxonomy-transform/examples/example-transform.js

@@ -0,0 +1,57 @@
+module.exports = async function ({ csv, taxonomy, fs }) {
+  const findParent = (data, index) => {
+    let notEmptyIndex = data[index].findIndex(
+      (c, index) => c != '' && index > 0
+    )
+
+    //Parent is concept scheme
+    if (notEmptyIndex == 3) {
+      return null
+    }
+
+    let parent = null
+
+    while (index > 1) {
+      index -= 1
+      if (data[index][notEmptyIndex - 2] != '') {
+        parent = data[index][0]
+        break
+      }
+    }
+
+    return parent
+  }
+
+  const csvFile = await fs.readFile(
+    fs.cwd() + '/docs/organization/taxonomy-transform/examples/example-data.csv'
+  )
+  const { data } = await csv.parse(csvFile.toString(), { header: false })
+  let conceptScheme = null
+  let parent = null
+
+  for (let i = 0; i < data.length; i++) {
+    const row = data[i]
+
+    //Cocneptscheme
+    if (row[1] != '') {
+      conceptScheme = taxonomy.addConceptScheme(row[0], {
+        prefLabel: row[1]
+      })
+    } else {
+      const notEmptyIndex = row.findIndex((c, index) => c != '' && index > 0)
+      let concept = taxonomy.getConcept(row[0])
+
+      if (!concept) {
+        concept = taxonomy.addConcept(row[0], {
+          prefLabel: row[notEmptyIndex]
+        })
+      }
+
+      conceptScheme.addConcept(row[0])
+      parent = findParent(data, i)
+      if (parent) {
+        concept.addBroader(parent)
+      }
+    }
+  }
+}

lib/cmds/organization_cmds/taxonomy-transform.ts

@@ -0,0 +1,201 @@
+/* eslint-disable @typescript-eslint/no-empty-function */
+import Listr from 'listr'
+import { noop } from 'lodash'
+import path from 'path'
+import type { Argv } from 'yargs'
+import { handleAsyncError as handle } from '../../utils/async'
+import { createPlainClient } from '../../utils/contentful-clients'
+import { cursorPaginate } from '../../utils/cursor-pagninate'
+import { ensureDir, getPath, readFileP, writeFileP } from '../../utils/fs'
+import { getHeadersFromOption } from '../../utils/headers'
+import { success, log } from '../../utils/log'
+import * as Papa from 'papaparse'
+import { Taxonomy } from './utils/taxonomy'
+
+module.exports.command = 'taxonomy-transform'
+
+module.exports.desc =
+  'transform taxonomy from external format to contentful json format'
+
+module.exports.builder = (yargs: Argv) => {
+  return yargs
+    .usage('Usage: contentful organization taxonomy-transform')
+    .option('management-token', {
+      alias: 'mt',
+      describe: 'Contentful management API token',
+      type: 'string'
+    })
+    .option('organization-id', {
+      alias: 'oid',
+      describe: 'ID of Organization with source data',
+      type: 'string',
+      demandOption: true
+    })
+    .option('header', {
+      alias: 'H',
+      type: 'string',
+      describe: 'Pass an additional HTTP Header'
+    })
+    .option('output-file', {
+      alias: 'o',
+      type: 'string',
+      describe:
+        'Output file. It defaults to ./data/<timestamp>-<organization-id>-transformed.json'
+    })
+    .option('transform-script', {
+      alias: 't',
+      describe: 'Script used to transform the taxonomy data',
+      type: 'string',
+      demandOption: true
+    })
+    .option('save-file', {
+      describe: 'Save the transformed taxonomies as a json file',
+      type: 'boolean',
+      default: true
+    })
+    .option('silent', {
+      alias: 's',
+      type: 'boolean',
+      describe: 'Suppress any log output',
+      default: false
+    })
+}
+
+interface Params {
+  context: { managementToken: string }
+  header?: string
+  organizationId: string
+  outputFile?: string
+  transformScript: string
+  saveFile?: boolean
+  silent?: boolean
+}
+
+export const defaultLocale = 'en-US'
+
+export interface TransformContext {
+  csv: {
+    // parses any CSV to a JSON
+    parse:
+      | (<T>(
+          csvString: string,
+          config?: Papa.ParseConfig
+        ) => Papa.ParseResult<T>)
+      | (<T>(
+          file: File,
+          config?: Papa.ParseConfig
+        ) => Promise<Papa.ParseResult<T>>)
+  }
+  fs: {
+    // reads any file from disc
+    readFile: typeof readFileP
+    cwd: typeof process.cwd
+  }
+  taxonomy: Taxonomy
+}
+
+const transformContext: TransformContext = {
+  csv: {
+    parse: Papa.parse
+  },
+  fs: {
+    readFile: readFileP,
+    cwd: process.cwd
+  },
+  taxonomy: new Taxonomy()
+}
+
+async function taxonomyTransform({
+  context,
+  header,
+  organizationId,
+  outputFile,
+  saveFile,
+  transformScript,
+  silent
+}: Params) {
+  const { managementToken } = context
+
+  const client = await createPlainClient({
+    accessToken: managementToken,
+    feature: 'taxonomy-transform',
+    headers: getHeadersFromOption(header),
+    throttle: 8,
+    logHandler: noop
+  })
+
+  const outputTarget = getPath(
+    outputFile ||
+      path.join('data', `${Date.now()}-${organizationId}-transformed.json`)
+  )
+  await ensureDir(path.dirname(outputTarget))
+
+  const tasks = new Listr(
+    [
+      {
+        title: 'Transforming taxonomy data',
+        task: async ctx => {
+          return new Listr([
+            {
+              title: 'Exporting Concepts',
+              task: async () => {
+                ctx.taxonomy.setExistingConcepts(
+                  await cursorPaginate({
+                    queryPage: pageUrl =>
+                      client.concept.getMany({
+                        organizationId,
+                        query: { pageUrl }
+                      })
+                  })
+                )
+              }
+            },
+            {
+              title: 'Exporting Concept Schemes',
+              task: async () => {
+                ctx.taxonomy.setExistingConceptSchemes(
+                  await cursorPaginate({
+                    queryPage: pageUrl =>
+                      client.conceptScheme.getMany({
+                        organizationId,
+                        query: { pageUrl }
+                      })
+                  })
+                )
+              }
+            },
+            {
+              title: 'Running transform script',
+              task: async () => {
+                const filePath = path.resolve(process.cwd(), transformScript)
+                const transform = await import(filePath)
+
+                await transform.default(ctx)
+              }
+            }
+          ])
+        }
+      }
+    ],
+    { renderer: silent ? 'silent' : 'default' }
+  )
+
+  await tasks.run(transformContext)
+
+  const result = transformContext.taxonomy.toJson()
+
+  if (saveFile) {
+    await writeFileP(outputTarget, JSON.stringify(result, null, 2))
+    !silent &&
+      success(`✅ Data transformed successfully and saved to ${outputTarget}`)
+  } else {
+    log(JSON.stringify(result, null, 2))
+    !silent && success(`✅ Data transformed successfully`)
+  }
+}
+
+module.exports.taxonomyTransform = taxonomyTransform
+
+module.exports.handler = handle(taxonomyTransform)
+
+export default taxonomyTransform