add css/scss modules support, back (#1174)

* add css/scss modules support, back

* add postcss back to dev client

* upgrade ci node

* appease linter

* new node openssl override

* doc fix

* mise var; yarnrc for integrity issues

* support node externals allowlist

* alpha publishes

* alpha publishes

* support webpackhot external on dev server

* bundle all css

* dev srcmap

* update to faster source map

* custom externals algorithm

* custom externals algorithm

* remove externals dep

* externals handle no deps in package.json

* externals handle bad package.json resolution in e2e tests

* webpack 4 bs

Author: delambo

.github/workflows/continuous-integration.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [18.x, 20.x]
 
     steps:
     - uses: actions/checkout@v2

.mise.toml

@@ -0,0 +1,9 @@
+[tools]
+node = '20.10.0'
+# stuck on this version of yarn because of some weird problems
+# related to babel tooling when installing new dependencies
+yarn = '1.19.0'
+
+[env]
+MISE_FETCH_REMOTE_VERSIONS_TIMEOUT="300 s"
+NODE_OPTIONS = "--openssl-legacy-provider"

.nvmrc

@@ -0,0 +1 @@
+unsafe-disable-integrity-migration false

.yarnrc

@@ -18,13 +18,21 @@ Please make sure all PRs are:
 
 If you want to propose a large feature idea or architecture change you should consider submitting an RFC. It's often helpful to get feedback on your concept in an issue before starting the RFC. RFCs are an evolving process in the `kyt` repository so expect a lot of changes and guidelines in the future. You can find the `kyt` RFC template [here](/rfc/template.md).
 
-## kyt local development
+## kyt local development workflow
 
-1. `nvm use`
 1. Fork and clone `kyt`
+1. [setup [mise](https://mise.jdx.dev/) and `mise install`]
 1. Run `yarn bootstrap` to install the packages in the monorepo
+1. Open a new shell and run `yarn watch`
 
-[lerna](https://github.com/lerna/lerna) is used to manage the monorepo but most of the development commands should be exercised through root directory `package.json` scripts. The following are some useful npm scripts for development:
+Most changes are best to develop against the universal starter kyt:
+
+1. `cd packages/kyt-starter-universal/starter-src`
+1. Run `yarn dev` or `yarn build` to test against kyt development and production builds
+
+Note: After you make changes, the watcher will update libraries but you will likely have to restart the universal app process to test changes. The watcher only works against kyt-core, server and runtime. Changes to babel presets and a few other packages may require you to re-`yarn bootstrap` or `yarn clean-bootstrap`. When in doubt run `yarn clean-bootstrap`.
+
+[lerna](https://github.com/lerna/lerna) is used to manage the monorepo but most of the development commands should be exercised through root directory `package.json` scripts.
 
 ### bootstrap
 
@@ -77,14 +85,12 @@ pushing directly to `main`.
 For more information on using `lerna` to publish, see [the `lerna publish`
 documentation](https://github.com/lerna/lerna/tree/main/commands/publish#readme).
 
-### Development Versions
+### Publishing Alpha Versions
 
-If you would like your prerelease to have the `next` dist tag, rather than
-`latest`, such as when creating a release candidate or testing a development
-version, you can use the provided `publish:next` script.
+If you would like to publish alpha release versions, for example `[email protected]`:
 
 ```sh
-$ GH_TOKEN=$GITHUB_TOKEN npm run publish:next
+$ GH_TOKEN=$GITHUB_TOKEN npm run publish:alpha
 ```
 
 If you need more functionality than this, it is recommended that you pass your

CONTRIBUTING.md

@@ -2,6 +2,14 @@
 
 # kyt
 
+### Deprecation notice
+
+This project is still used internally at the new york times but has long been deprecated. Please don't expect any support or documented releases going forward.
+
+---
+
+---
+
 Every sizable JavaScript web app needs a common foundation: a setup to build, run, test and lint your code. `kyt` is a toolkit that encapsulates and manages the configuration for web apps.
 
 Read more about kyt in our [blog post](https://open.nytimes.com/introducing-kyt-our-web-app-configuration-toolkit-9ccddf6f6988).

README.md

@@ -26,9 +26,12 @@ describe('KYT CLI', () => {
     it(`sets up a ${slug} starter-kyt`, () => {
       const exec = new Promise(resolve => {
         shell.cd(rootPath);
-        const child = shell.exec('../packages/kyt-core/lib/index.js setup', (code, stdout) => {
-          resolve({ code, output: stdout });
-        });
+        const child = shell.exec(
+          'NODE_OPTIONS=--openssl-legacy-provider ../packages/kyt-core/lib/index.js setup',
+          (code, stdout) => {
+            resolve({ code, output: stdout });
+          }
+        );
         let skdone = false;
         let chooseDone = false;
         let ypmDone = false;

e2e_tests/tests/cli.test.js

@@ -10,7 +10,9 @@ describe('kyt build', () => {
   it('should compile files into a build directory', () => {
     util.setupStageWithFixture(stageName, 'build-default');
 
-    const output = shell.exec('../packages/kyt-core/lib/index.js build');
+    const output = shell.exec(
+      'NODE_OPTIONS=--openssl-legacy-provider ../packages/kyt-core/lib/index.js build'
+    );
 
     expect(shell.test('-f', 'build/publicAssets.json')).toBe(true);
     expect(shell.test('-d', 'build/server')).toBe(true);
@@ -46,15 +48,15 @@ describe('kyt build', () => {
 
   it('should ignore server build if hasServer=false', () => {
     util.setupStageWithFixture(stageName, 'build-no-server');
-    const output = shell.exec('npm run build');
+    const output = shell.exec('NODE_OPTIONS=--openssl-legacy-provider npm run build');
 
     expect(output.code).toBe(0);
     expect(shell.test('-d', 'build/server')).toBe(false);
   });
 
   it('should ignore client build if hasClient=false', () => {
     util.setupStageWithFixture(stageName, 'build-no-client');
-    const output = shell.exec('npm run build');
+    const output = shell.exec('NODE_OPTIONS=--openssl-legacy-provider npm run build');
 
     expect(output.code).toBe(0);
     expect(shell.test('-d', 'build/server')).toBe(true);

e2e_tests/tests/kyt-build.test.js

@@ -19,7 +19,7 @@ describe('starter kyts', () => {
     it('should start a dev server on :3000', () => {
       let outputTest;
       const run = new Promise(resolve => {
-        const child = shell.exec(`${kytCli} dev`, () => {
+        const child = shell.exec(`NODE_OPTIONS=--openssl-legacy-provider ${kytCli} dev`, () => {
           resolve(outputTest);
         });
         child.stdout.on('data', data => {
@@ -38,11 +38,14 @@ describe('starter kyts', () => {
 
     it('should build and run', () => {
       let outputTest;
-      shell.exec(`${kytCli} build`);
+      shell.exec(`NODE_OPTIONS=--openssl-legacy-provider ${kytCli} build`);
       const run = new Promise(resolve => {
-        const child = shell.exec('node build/server/main.js', () => {
-          resolve(outputTest);
-        });
+        const child = shell.exec(
+          'NODE_OPTIONS=--openssl-legacy-provider node build/server/main.js',
+          () => {
+            resolve(outputTest);
+          }
+        );
         child.stdout.on('data', data => {
           if (data.includes('✅  server started on port: 3000')) {
             shell.exec('sleep 5');
@@ -71,7 +74,7 @@ describe('starter kyts', () => {
     it('should start a server on :3001', () => {
       let outputTest;
       const run = new Promise(resolve => {
-        const child = shell.exec(`${kytCli} dev`, () => {
+        const child = shell.exec(`NODE_OPTIONS=--openssl-legacy-provider ${kytCli} dev`, () => {
           resolve(outputTest);
         });
         child.stdout.on('data', data => {
@@ -87,7 +90,7 @@ describe('starter kyts', () => {
     });
 
     it('should build', () => {
-      const output = shell.exec(`${kytCli} build`);
+      const output = shell.exec(`NODE_OPTIONS=--openssl-legacy-provider ${kytCli} build`);
 
       expect(output.stdout).toContain('✅  Done building');
 

e2e_tests/tests/starter-kyt.test.js

@@ -15,11 +15,11 @@
     "yarn": "^1"
   },
   "scripts": {
-    "bootstrap": "yarn && lerna run prepare",
+    "bootstrap": "yarn && lerna bootstrap && lerna run prepare && lerna link",
     "bootstrap:ci": "yarn && lerna run prepare",
     "clean-bootstrap": "lerna clean --yes && rm -rf node_modules && yarn bootstrap",
     "publish": "lerna publish",
-    "publish:next": "lerna publish --preid next --dist-tag next",
+    "publish:alpha": "lerna publish --canary --force-publish --preid alpha --dist-tag alpha",
     "test:starter": "lerna exec --scope \"kyt-starter-{server,static,universal}-src\" --",
     "test": "lerna run prepare && jest --detectOpenHandles && yarn test:starter yarn test --detectOpenHandles",
     "test:ci": "jest --detectOpenHandles --ci && yarn test:starter yarn test --detectOpenHandles --ci",
@@ -31,7 +31,10 @@
     "lint-staged": "yarn run lint -o",
     "lint-fix": "ESLINT_FIX=1 ESLINT_QUIET=1 yarn run lint",
     "lint:ci": "ESLINT_QUIET=1 yarn run lint",
-    "watch-cli": "yarn workspace kyt watch",
+    "watch": "run-p watch-*",
+    "watch-kyt": "yarn workspace kyt watch",
+    "watch-kyt-runtime": "yarn workspace kyt-runtime watch",
+    "watch-kyt-utils": "yarn workspace kyt-utils watch",
     "prepare": "husky install"
   },
   "devDependencies": {
@@ -62,6 +65,7 @@
     "jest-runner-eslint": "0.10.1",
     "jest-silent-reporter": "0.5.0",
     "lerna": "4.0.0",
+    "npm-run-all": "4.1.5",
     "prettier": "2.8.8",
     "react": "17.0.2",
     "react-dom": "17.0.2",

package.json

@@ -23,6 +23,7 @@
     "babel-preset-kyt-core": "2.0.1",
     "commander": "8.2.0",
     "core-js": "3.18.0",
+    "css-loader": "5.2.7",
     "file-loader": "6.2.0",
     "filesize": "8.0.3",
     "find-babel-config": "1.2.0",
@@ -36,23 +37,28 @@
     "kyt-utils": "1.3.33",
     "lodash.clonedeep": "4.5.0",
     "lodash.merge": "4.6.2",
+    "mini-css-extract-plugin": "1.6.2",
+    "optimize-css-assets-webpack-plugin": "6.0.1",
+    "postcss-loader": "4.2.0",
     "prop-types": "15.7.2",
     "ps-tree": "1.2.0",
     "react": "17.0.2",
     "react-dev-utils": "11.0.4",
     "react-error-overlay": "6.0.9",
     "regenerator-runtime": "0.13.9",
+    "sass": "1.74.1",
+    "sass-loader": "10.2.0",
     "semver": "7.3.5",
     "shelljs": "0.8.5",
     "simple-git": "2.42.0",
     "sockjs-client": "1.5.2",
     "source-map-support": "0.5.20",
     "strip-ansi": "6.0.0",
+    "style-loader": "0.23.1",
     "url-loader": "4.1.1",
     "webpack": "4.46.0",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "4.2.2",
-    "webpack-node-externals": "2.5.2",
     "webpackbar": "4.0.0"
   },
   "devDependencies": {

packages/kyt-core/package.json

@@ -0,0 +1,53 @@
+// Works similarly to webpack-node-externals but it's not as
+// aggressive. It only excludes top-level dependency-declared
+// modules. Respects an `allowList` of regexp's that will be
+// used to match modules to include in the bundle.
+const { userPackageJSONPath } = require('kyt-utils/paths')();
+
+let pkg;
+
+try {
+  // eslint-disable-next-line import/no-dynamic-require, global-require
+  pkg = require(userPackageJSONPath);
+} catch (e) {
+  pkg = {};
+}
+
+module.exports = (allowList = []) => {
+  // Get all of the dependencies from the package.json
+  // and filter out the ones that are in the allowList
+  const packageModules = [];
+  Object.keys(pkg.dependencies || []).forEach(module => {
+    allowList.forEach(allowedModule => {
+      if (!allowedModule.test(module)) packageModules.push(module);
+    });
+  });
+
+  return [
+    // eslint-disable-next-line consistent-return
+    (context, request, callback) => {
+      function getModuleName(requested) {
+        const scopedModuleRegex = new RegExp(
+          '@[a-zA-Z0-9][\\w-.]+/[a-zA-Z0-9][\\w-.]+([a-zA-Z0-9./]+)?',
+          'g'
+        );
+        const req = requested;
+        const delimiter = '/';
+
+        // Check if scoped module. For example: @company/boring-module
+        if (scopedModuleRegex.test(req)) {
+          // reset regexp
+          scopedModuleRegex.lastIndex = 0;
+          return req.split(delimiter, 2).join(delimiter);
+        }
+        return req.split(delimiter)[0];
+      }
+      if (packageModules.includes(getModuleName(request))) {
+        // Mark this module as EXTERNAL
+        return callback(null, `commonjs ${request}`);
+      }
+      // Include module in bundle / NOT EXTERNAL
+      callback();
+    },
+  ];
+};

packages/kyt-core/src/config/externals.js

@@ -0,0 +1,5 @@
+module.exports = {
+  postcssOptions: {
+    plugins: [['autoprefixer', {}]],
+  },
+};

packages/kyt-core/src/config/postcss.config.js

@@ -1,10 +1,13 @@
 // Development webpack config for client code
 const path = require('path');
 const webpack = require('webpack');
+const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const errorOverlayMiddleware = require('react-dev-utils/errorOverlayMiddleware');
 const { kytWebpackPlugins } = require('kyt-runtime/webpack');
-const { clientSrcPath, assetsBuildPath, publicBuildPath } = require('kyt-utils/paths')();
+const { clientSrcPath, assetsBuildPath, publicBuildPath, publicSrcPath } =
+  require('kyt-utils/paths')();
 const getPolyfill = require('./getPolyfill');
+const postcssLoader = require('../utils/getPostcssLoader');
 
 module.exports = options => {
   const main = [
@@ -65,6 +68,40 @@ module.exports = options => {
       },
     },
 
-    plugins: [...kytWebpackPlugins(options), new webpack.HotModuleReplacementPlugin()],
+    module: {
+      rules: [
+        {
+          test: /\.module\.(sc|c)ss$/,
+          use: [
+            // 'style-loader',
+            MiniCssExtractPlugin.loader,
+            {
+              loader: 'css-loader',
+              options: {
+                sourceMap: true,
+                modules: {
+                  localIdentName: '[name]-[local]--[hash:base64:5]',
+                  // exportOnlyLocals: true,
+                },
+              },
+            },
+            postcssLoader,
+            'sass-loader',
+          ],
+          exclude: [publicSrcPath],
+        },
+      ],
+    },
+
+    plugins: [
+      ...kytWebpackPlugins(options),
+      new MiniCssExtractPlugin({
+        // Options similar to the same options in webpackOptions.output
+        // both options are optional
+        filename: '[name].css',
+        chunkFilename: '[id].css',
+      }),
+      new webpack.HotModuleReplacementPlugin(),
+    ],
   };
 };