Merge pull request #287 from ibi-group/check-i18n-ymls

Check i18n ymls

Author: binh-dam-ibigroup

Diff for: package.json

@@ -17,6 +17,7 @@
     "@babel/preset-env": "^7.10",
     "@babel/preset-react": "^7.10",
     "@babel/preset-typescript": "^7.13.0",
+    "@formatjs/cli": "^4.2.32",
     "@semantic-release/git": "^9.0.0",
     "@storybook/addon-a11y": "^6.4.19",
     "@storybook/addon-actions": "^6.4.19",
@@ -58,6 +59,7 @@
     "jest-resolve": "^24.8.0",
     "jest-styled-components": "^7.0.5",
     "jest-yaml-transform": "^0.2.0",
+    "js-yaml": "^4.1.0",
     "leaflet": "^1.6.0",
     "lerna": "^3.18.4",
     "lint-staged": "^8.2.0",
@@ -90,6 +92,7 @@
     "bootstrap": "lerna bootstrap --use-workspaces",
     "build:cjs": "lerna exec --parallel -- babel --extensions '.js,.ts,.tsx' --ignore **/*.story.js,**/*.story.ts,**/*.story.d.ts,**/*.story.tsx,**/*.spec.js,**/*.spec.ts,**/*.test.js,**/*.test.ts,**/__tests__/**,**/__unpublished__/** --root-mode upward --source-maps true src -d lib",
     "build:esm": "lerna exec --parallel -- cross-env BABEL_ENV=esm babel --extensions '.js,.ts,.tsx' --ignore **/*.story.js,**/*.story.ts,**/*.story.d.ts,**/*.story.tsx,**/*.spec.js,**/*.spec.ts,**/*.test.js,**/*.test.ts,**/__tests__/**,**/__unpublished__/** --root-mode upward --source-maps true src -d esm",
+    "check:i18n": "node packages/scripts/lib/validate-i18n.js packages/**/src packages/**/i18n",
     "prepublish": "yarn typescript && yarn build:cjs && yarn build:esm",
     "check-eslint-config": "eslint --print-config jestconfig.js | eslint-config-prettier-check",
     "coverage": "jest --coverage",
@@ -102,7 +105,7 @@
     "lint": "yarn lint:js && yarn lint:styles",
     "prettier": "prettier --write \"**/*.{json,md,yml}\"",
     "semantic-release": "lerna exec --concurrency 1 -- semantic-release -e semantic-release-monorepo",
-    "test": "yarn lint:js && yarn lint:styles && yarn typescript && yarn unit && yarn a11y-test",
+    "test": "yarn lint:js && yarn lint:styles && yarn check:i18n && yarn typescript && yarn unit && yarn a11y-test",
     "typescript": "lerna run tsc",
     "unit": "jest --testPathIgnorePatterns a11y",
     "update-internal-dependencies": "node scripts/update-internal-dependencies.js"

Diff for: packages/location-field/src/index.tsx

@@ -593,6 +593,7 @@ const LocationField = ({
             isActive={itemIndex === activeIndex}
             key={optionKey++}
             onClick={locationSelected}
+            // @ts-ignore Fixed in another PR
             title={coreUtils.map.formatStoredPlaceName(userLocation)}
           />
         );

Diff for: packages/scripts/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@opentripplanner/scripts",
+  "version": "1.0.0",
+  "description": "Utility scripts for the OTP-UI library",
+  "main": "lib/index.js",
+  "module": "esm/index.js",
+  "types": "lib/index.d.js",
+  "repository": "https://github.com/opentripplanner/otp-ui.git",
+  "homepage": "https://github.com/opentripplanner/otp-ui#readme",
+  "author": "Binh Dam",
+  "license": "MIT",
+  "private": false,
+  "dependencies": {
+    "@formatjs/cli": "^4.2.33",
+    "flat": "^5.0.2",
+    "glob": "^8.0.3",
+    "glob-promise": "^4.2.2",
+    "js-yaml": "^4.1.0"
+  },
+  "scripts": {
+    "collect-i18n": "node lib/collect-i18n-messages.js ../**/src ../**/i18n/en-US.yml > i18n.csv",
+    "tsc": "tsc",
+    "validate-i18n": "node lib/validate-i18n.js ../**/src ../**/i18n"
+  },
+  "bugs": {
+    "url": "https://github.com/opentripplanner/otp-ui/issues"
+  },
+  "gitHead": "0af1b7cda60bd4252b219dcf893e01c2acb2ed5d"
+}

Diff for: packages/scripts/src/collect-i18n-messages.ts

@@ -0,0 +1,77 @@
+/* eslint-disable no-console */
+/**
+ * This script collects message ids gathered by the formatjs extract command in the specified files and folder(s)
+ * and creates a CSV file with the id, description, and messages in the selected language(s).
+ * This script is shipped as part of a package so it can be used in other code bases as needed.
+ */
+// Example usage for all packages and all languages in this repo:
+//   node path-to/lib/collect-i18n-messages.js ../**/src ../**/i18n
+// Example usage for all packages and one language in this repo:
+//   node path-to-lib/collect-i18n-messages.js ../**/src ../**/i18n/en-US.yml
+
+import { extract } from "@formatjs/cli";
+import flatten from "flat";
+
+import { isNotSpecialId, loadYamlFile, sortSourceAndYmlFiles } from "./util";
+
+// The data that corresponds to rows in the CSV output.
+type MessageData = Record<
+  string,
+  Record<string, string> & {
+    description: string;
+  }
+>;
+
+/**
+ * Collect all messages and create a formatted output.
+ */
+async function collectAndPrintOutMessages({ sourceFiles, ymlFilesByLocale }) {
+  // Gather message ids from code.
+  const messagesFromCode = JSON.parse(await extract(sourceFiles, {}));
+  const messageIdsFromCode = Object.keys(messagesFromCode);
+  const allLocales = Object.keys(ymlFilesByLocale);
+
+  // CSV heading
+  console.log(`ID,Description,${allLocales.join(",")}`);
+
+  // Will contain id, description, and a column for each language.
+  const messageData: MessageData = {};
+
+  // For each locale, check that all ids in messages are in the yml files.
+  // Accessorily, log message ids from yml files that are not used in the code.
+  await Promise.all(
+    allLocales.map(async locale => {
+      const allI18nPromises = ymlFilesByLocale[locale].map(loadYamlFile);
+      const allI18nMessages = await Promise.all(allI18nPromises);
+      let allI18nMessagesFlattened = {};
+
+      allI18nMessages.forEach(i18nMessages => {
+        const flattenedMessages: Record<string, string> = flatten(i18nMessages);
+        allI18nMessagesFlattened = {
+          ...allI18nMessagesFlattened,
+          ...flattenedMessages
+        };
+      });
+
+      messageIdsFromCode.filter(isNotSpecialId).forEach(id => {
+        const { description } = messagesFromCode[id];
+        const message = allI18nMessagesFlattened[id]?.trim() || undefined;
+
+        if (!messageData[id]) {
+          messageData[id] = {
+            description
+          };
+        }
+        messageData[id][locale] = message;
+      });
+    })
+  );
+
+  Object.keys(messageData).forEach(id => {
+    const row = messageData[id];
+    const messages = allLocales.map(locale => row[locale]);
+    console.log(`${id},"${row.description}","${messages}"`);
+  });
+}
+
+sortSourceAndYmlFiles(process.argv).then(collectAndPrintOutMessages);

Diff for: packages/scripts/src/util.ts

@@ -0,0 +1,89 @@
+import { promises as fs } from "fs";
+import { load } from "js-yaml";
+import glob from "glob-promise";
+import path from "path";
+
+export interface SourceFilesAndYmlFilesByLocale {
+  sourceFiles: string[];
+  ymlFilesByLocale: Record<string, string>;
+}
+
+function shouldProcessFile(fileName: string): boolean {
+  return (
+    !fileName.includes("/__") &&
+    !fileName.includes("node_modules") &&
+    !fileName.endsWith(".d.ts")
+  );
+}
+
+/**
+ * @returns true if the id is not special or reserved (i.e. doesn't start with "_").
+ */
+export function isNotSpecialId(id: string): boolean {
+  return !id.startsWith("_");
+}
+
+/**
+ * Helper function that sorts yml and source files into two buckets.
+ * @param argv The value from process.argv.
+ * @returns A composite object with a list for yml files by locale, and a list for source files.
+ */
+export async function sortSourceAndYmlFiles(argv: string[]) {
+  const sourceFiles = [];
+  const ymlFilesByLocale = {};
+
+  // Places the give file into the source or yml file bucket above.
+  function sortFile(fileName: string): void {
+    const parsedArg = path.parse(fileName);
+    if (parsedArg.ext === ".yml") {
+      const locale = parsedArg.name;
+      if (!ymlFilesByLocale[locale]) {
+        ymlFilesByLocale[locale] = [];
+      }
+      const ymlFilesForLocale = ymlFilesByLocale[locale];
+      if (!ymlFilesForLocale.includes(fileName)) {
+        ymlFilesForLocale.push(fileName);
+      }
+      ymlFilesByLocale[locale].push(fileName);
+    } else if (!sourceFiles.includes(fileName)) {
+      sourceFiles.push(fileName);
+    }
+  }
+
+  // Note: reminder that node.js provides the first two argv values:
+  // - argv[0] is the name of the executable file.
+  // - argv[1] is the path to the script file.
+  // - argv[2] and beyond are the folders passed to the script.
+  const allGlobPromises = [];
+  for (let i = 2; i < argv.length; i++) {
+    // List the files recursively (glob) for this folder.
+    const arg = argv[i];
+
+    // If argument ends with .yml, treat as a file.
+    if (arg.endsWith(".yml")) {
+      sortFile(arg);
+    } else {
+      // Otherwise, it is a folder, and use glob to get files recursively.
+      // For glob argument info, see their docs at https://github.com/ahmadnassri/node-glob-promise#api.
+      allGlobPromises.push(glob(`${arg}/**/*.{{j,t}s{,x},yml}`));
+    }
+  }
+
+  const allFileLists = await Promise.all(allGlobPromises);
+  allFileLists.forEach(files =>
+    files.filter(shouldProcessFile).forEach(sortFile)
+  );
+
+  return {
+    sourceFiles,
+    ymlFilesByLocale
+  };
+}
+
+/**
+ * Load yaml from a file into a js object
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function loadYamlFile(filename: string): Promise<any> {
+  return load(await fs.readFile(filename));
+}

Diff for: packages/scripts/src/validate-i18n.ts

@@ -0,0 +1,86 @@
+/* eslint-disable no-console */
+/**
+ * This script checks that message ids gathered by the formatjs extract command
+ * are present in the specified folder(s).
+ * It will produce an error code if message ids are present in a language but not another,
+ * or if message ids are in a i18n yml files but not in the code or vice-versa.
+ * This script is shipped as part of a package so it can be used in other code bases as needed.
+ */
+// Example usage for one package in this repo:
+//   node path-to/lib/validate-i18n.js ../trip-details/src ../trip-details/i18n
+// Example usage for all packages in this repo:
+//   node path-to/lib/validate-i18n.js ../**/src ../**/i18n
+
+const { extract } = require("@formatjs/cli");
+const flatten = require("flat");
+
+const {
+  isNotSpecialId,
+  loadYamlFile,
+  sortSourceAndYmlFiles
+} = require("./util");
+
+/**
+ * Checks message ids completeness between code and yml files for all locales in repo.
+ */
+async function checkI18n({ sourceFiles, ymlFilesByLocale }) {
+  // Gather message ids from code.
+  const messagesFromCode = JSON.parse(await extract(sourceFiles, {}));
+  const messageIdsFromCode = Object.keys(messagesFromCode);
+  console.log(
+    `Checking ${messageIdsFromCode.length} strings from ${
+      Object.keys(ymlFilesByLocale["en-US"]).length
+    } message files against ${sourceFiles.length} source files.`
+  );
+  let errorCount = 0;
+
+  // For each locale, check that all ids in messages are in the yml files.
+  // Accessorily, log message ids from yml files that are not used in the code.
+  await Promise.all(
+    Object.keys(ymlFilesByLocale).map(async locale => {
+      const idsChecked = [];
+      const idsNotInCode = [];
+
+      const allI18nPromises = ymlFilesByLocale[locale].map(loadYamlFile);
+      const allI18nMessages = await Promise.all(allI18nPromises);
+
+      allI18nMessages.forEach(i18nMessages => {
+        const flattenedMessages = flatten(i18nMessages);
+
+        // Message ids from code must be present in yml.
+        messageIdsFromCode
+          .filter(id => flattenedMessages[id])
+          .forEach(id => idsChecked.push(id));
+
+        // Message ids from yml (except those starting with "_") must be present in code.
+        Object.keys(flattenedMessages)
+          .filter(isNotSpecialId)
+          .filter(id => !messageIdsFromCode.includes(id))
+          .forEach(id => idsNotInCode.push(id));
+      });
+
+      // Collect ids in code not found in yml.
+      const missingIdsForLocale = messageIdsFromCode.filter(
+        id => !idsChecked.includes(id)
+      );
+
+      // Print errors.
+      missingIdsForLocale.forEach(id => {
+        console.error(`Message '${id}' is missing from locale ${locale}.`);
+      });
+      idsNotInCode.forEach(id => {
+        console.error(
+          `Message '${id}' from locale ${locale} is not used in code.`
+        );
+      });
+      errorCount += missingIdsForLocale.length + idsNotInCode.length;
+    })
+  );
+
+  console.log(`There were ${errorCount} error(s).`);
+  if (errorCount > 0) {
+    process.exit(1);
+  }
+}
+
+sortSourceAndYmlFiles(process.argv).then(checkI18n);

Diff for: packages/scripts/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./lib"
+  },
+  "include": ["src/**/*"]
+}